forked from TrueCloudLab/frostfs-s3-gw
Compare commits
2 commits
master
...
KirillovDe
Author | SHA1 | Date | |
---|---|---|---|
4bf929dde3 | |||
717f1a96ab |
228 changed files with 8942 additions and 28286 deletions
|
@ -1,7 +1,7 @@
|
|||
FROM golang:1.21 as builder
|
||||
FROM golang:1.19 as builder
|
||||
|
||||
ARG BUILD=now
|
||||
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
||||
ARG REPO=github.com/TrueCloudLab/frostfs-s3-gw
|
||||
ARG VERSION=dev
|
||||
|
||||
WORKDIR /src
|
||||
|
|
|
@ -1,23 +0,0 @@
|
|||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
builds:
|
||||
name: Builds
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.20', '1.21' ]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '${{ matrix.go_versions }}'
|
||||
|
||||
- name: Build binary
|
||||
run: make
|
||||
|
||||
- name: Check dirty suffix
|
||||
run: if [[ $(make version) == *"dirty"* ]]; then echo "Version has dirty suffix" && exit 1; fi
|
|
@ -1,20 +0,0 @@
|
|||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
dco:
|
||||
name: DCO
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Run commit format checker
|
||||
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3
|
||||
with:
|
||||
from: 'origin/${{ github.event.pull_request.base.ref }}'
|
|
@ -1,41 +0,0 @@
|
|||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.21'
|
||||
cache: true
|
||||
|
||||
- name: Install linters
|
||||
run: make lint-install
|
||||
|
||||
- name: Run linters
|
||||
run: make lint
|
||||
|
||||
tests:
|
||||
name: Tests
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.20', '1.21' ]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '${{ matrix.go_versions }}'
|
||||
|
||||
- name: Update Go modules
|
||||
run: make dep
|
||||
|
||||
- name: Run tests
|
||||
run: make test
|
|
@ -1,21 +0,0 @@
|
|||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
vulncheck:
|
||||
name: Vulncheck
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Install govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
|
||||
- name: Run govulncheck
|
||||
run: govulncheck ./...
|
2
.github/CODEOWNERS
vendored
2
.github/CODEOWNERS
vendored
|
@ -1 +1 @@
|
|||
* @alexvanin @dkirillov
|
||||
* @alexvanin @KirillovDenis
|
||||
|
|
45
.github/ISSUE_TEMPLATE/bug_report.md
vendored
45
.github/ISSUE_TEMPLATE/bug_report.md
vendored
|
@ -1,45 +0,0 @@
|
|||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: community, triage, bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
<!--- If you're describing a bug, tell us what should happen -->
|
||||
<!--- If you're suggesting a change/improvement, tell us how it should work -->
|
||||
|
||||
## Current Behavior
|
||||
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
|
||||
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
||||
|
||||
## Possible Solution
|
||||
<!--- Not obligatory -->
|
||||
<!--- If no reason/fix/additions for the bug can be suggested, -->
|
||||
<!--- uncomment the following phrase: -->
|
||||
|
||||
<!--- No fix can be suggested by a QA engineer. Further solutions shall be up to developers. -->
|
||||
|
||||
## Steps to Reproduce (for bugs)
|
||||
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||
<!--- reproduce this bug. -->
|
||||
|
||||
1.
|
||||
|
||||
## Context
|
||||
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
||||
<!--- Providing context helps us come up with a solution that is most useful in the real world -->
|
||||
|
||||
## Regression
|
||||
<!-- Is this issue a regression? (Yes / No) -->
|
||||
<!-- If Yes, optionally please include version or commit id or PR# that caused this regression, if you have these details. -->
|
||||
|
||||
## Your Environment
|
||||
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
||||
* Version used:
|
||||
* Server setup and configuration:
|
||||
* Operating System and version (`uname -a`):
|
1
.github/ISSUE_TEMPLATE/config.yml
vendored
1
.github/ISSUE_TEMPLATE/config.yml
vendored
|
@ -1 +0,0 @@
|
|||
blank_issues_enabled: false
|
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
|
@ -1,20 +0,0 @@
|
|||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: community, triage
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## Is your feature request related to a problem? Please describe.
|
||||
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
|
||||
|
||||
## Describe the solution you'd like
|
||||
<!--- A clear and concise description of what you want to happen. -->
|
||||
|
||||
## Describe alternatives you've considered
|
||||
<!--- A clear and concise description of any alternative solutions or features you've considered. -->
|
||||
|
||||
## Additional context
|
||||
<!--- Add any other context or screenshots about the feature request here. -->
|
73
.github/workflows/builds.yml
vendored
Normal file
73
.github/workflows/builds.yml
vendored
Normal file
|
@ -0,0 +1,73 @@
|
|||
name: Builds
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- 'support/*'
|
||||
types: [ opened, synchronize ]
|
||||
paths-ignore:
|
||||
- '**/*.md'
|
||||
|
||||
jobs:
|
||||
build_cli:
|
||||
name: Build CLI
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.19
|
||||
|
||||
- name: Restore Go modules from cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: /home/runner/go/pkg/mod
|
||||
key: deps-${{ hashFiles('go.sum') }}
|
||||
|
||||
- name: Get tree-service client
|
||||
run: make sync-tree
|
||||
|
||||
- name: Update Go modules
|
||||
run: make dep
|
||||
|
||||
- name: Build CLI
|
||||
run: make
|
||||
|
||||
- name: Check version
|
||||
run: if [[ $(make version) == *"dirty"* ]]; then exit 1; fi
|
||||
|
||||
build_image:
|
||||
needs: build_cli
|
||||
name: Build Docker image
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.19
|
||||
|
||||
- name: Restore Go modules from cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: /home/runner/go/pkg/mod
|
||||
key: deps-${{ hashFiles('go.sum') }}
|
||||
|
||||
- name: Get tree-service client
|
||||
run: make sync-tree
|
||||
|
||||
- name: Update Go modules
|
||||
run: make dep
|
||||
|
||||
- name: Build Docker image
|
||||
run: make image
|
67
.github/workflows/codeql-analysis.yml
vendored
Normal file
67
.github/workflows/codeql-analysis.yml
vendored
Normal file
|
@ -0,0 +1,67 @@
|
|||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, 'support/*' ]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [ master, 'support/*' ]
|
||||
schedule:
|
||||
- cron: '35 8 * * 1'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [ 'go' ]
|
||||
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
|
||||
# Learn more:
|
||||
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v2
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
|
||||
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
||||
# and modify them (or add more) to build your code if your project
|
||||
# uses a compiled language
|
||||
|
||||
#- run: |
|
||||
# make bootstrap
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v2
|
22
.github/workflows/dco.yml
vendored
Normal file
22
.github/workflows/dco.yml
vendored
Normal file
|
@ -0,0 +1,22 @@
|
|||
name: DCO check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- 'support/*'
|
||||
|
||||
jobs:
|
||||
commits_check_job:
|
||||
runs-on: ubuntu-latest
|
||||
name: Commits Check
|
||||
steps:
|
||||
- name: Get PR Commits
|
||||
id: 'get-pr-commits'
|
||||
uses: tim-actions/get-pr-commits@master
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: DCO Check
|
||||
uses: tim-actions/dco@master
|
||||
with:
|
||||
commits: ${{ steps.get-pr-commits.outputs.commits }}
|
96
.github/workflows/tests.yml
vendored
Normal file
96
.github/workflows/tests.yml
vendored
Normal file
|
@ -0,0 +1,96 @@
|
|||
name: Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- 'support/*'
|
||||
types: [opened, synchronize]
|
||||
paths-ignore:
|
||||
- '**/*.md'
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Get tree-service client
|
||||
run: make sync-tree
|
||||
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
|
||||
cover:
|
||||
name: Coverage
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
env:
|
||||
CGO_ENABLED: 1
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.19
|
||||
|
||||
- name: Restore Go modules from cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: /home/runner/go/pkg/mod
|
||||
key: deps-${{ hashFiles('go.sum') }}
|
||||
|
||||
- name: Get tree-service client
|
||||
run: make sync-tree
|
||||
|
||||
- name: Update Go modules
|
||||
run: make dep
|
||||
|
||||
- name: Test and write coverage profile
|
||||
run: make cover
|
||||
|
||||
- name: Upload coverage results to Codecov
|
||||
uses: codecov/codecov-action@v1
|
||||
with:
|
||||
fail_ci_if_error: false
|
||||
path_to_write_report: ./coverage.txt
|
||||
verbose: true
|
||||
|
||||
tests:
|
||||
name: Tests
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.18.x', '1.19.x' ]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: '${{ matrix.go_versions }}'
|
||||
|
||||
- name: Restore Go modules from cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: /home/runner/go/pkg/mod
|
||||
key: deps-${{ hashFiles('go.sum') }}
|
||||
|
||||
- name: Get tree-service client
|
||||
run: make sync-tree
|
||||
|
||||
- name: Update Go modules
|
||||
run: make dep
|
||||
|
||||
- name: Run tests
|
||||
run: make test
|
4
.gitignore
vendored
4
.gitignore
vendored
|
@ -2,6 +2,9 @@
|
|||
.idea
|
||||
.vscode
|
||||
|
||||
# Tree service
|
||||
internal/frostfs/services/tree/
|
||||
|
||||
# Vendoring
|
||||
vendor
|
||||
|
||||
|
@ -25,3 +28,4 @@ debian/files
|
|||
debian/*.log
|
||||
debian/*.substvars
|
||||
debian/frostfs-s3-gw/
|
||||
|
||||
|
|
11
.gitlint
11
.gitlint
|
@ -1,11 +0,0 @@
|
|||
[general]
|
||||
fail-without-commits=True
|
||||
regex-style-search=True
|
||||
contrib=CC1
|
||||
|
||||
[title-match-regex]
|
||||
regex=^\[\#[0-9Xx]+\]\s
|
||||
|
||||
[ignore-by-title]
|
||||
regex=^Release(.*)
|
||||
ignore=title-match-regex
|
|
@ -4,7 +4,7 @@
|
|||
# options for analysis running
|
||||
run:
|
||||
# timeout for analysis, e.g. 30s, 5m, default is 1m
|
||||
timeout: 15m
|
||||
timeout: 5m
|
||||
|
||||
# include test files or not, default is true
|
||||
tests: true
|
||||
|
@ -24,16 +24,6 @@ linters-settings:
|
|||
govet:
|
||||
# report about shadowed variables
|
||||
check-shadowing: false
|
||||
custom:
|
||||
truecloudlab-linters:
|
||||
path: bin/external_linters.so
|
||||
original-url: git.frostfs.info/TrueCloudLab/linters.git
|
||||
settings:
|
||||
noliteral:
|
||||
enable: true
|
||||
target-methods: ["Fatal"]
|
||||
disable-packages: ["codes", "tc"]
|
||||
constants-package: "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
|
||||
linters:
|
||||
enable:
|
||||
|
@ -55,7 +45,6 @@ linters:
|
|||
- gofmt
|
||||
- whitespace
|
||||
- goimports
|
||||
- truecloudlab-linters
|
||||
disable-all: true
|
||||
fast: false
|
||||
|
||||
|
|
|
@ -1,52 +0,0 @@
|
|||
ci:
|
||||
autofix_prs: false
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/jorisroovers/gitlint
|
||||
rev: v0.19.1
|
||||
hooks:
|
||||
- id: gitlint
|
||||
stages: [commit-msg]
|
||||
- id: gitlint-ci
|
||||
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.4.0
|
||||
hooks:
|
||||
- id: check-added-large-files
|
||||
- id: check-case-conflict
|
||||
- id: check-executables-have-shebangs
|
||||
- id: check-shebang-scripts-are-executable
|
||||
- id: check-merge-conflict
|
||||
- id: check-json
|
||||
- id: check-xml
|
||||
- id: check-yaml
|
||||
- id: trailing-whitespace
|
||||
args: [--markdown-linebreak-ext=md]
|
||||
- id: end-of-file-fixer
|
||||
exclude: ".key$"
|
||||
|
||||
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||
rev: v0.9.0.2
|
||||
hooks:
|
||||
- id: shellcheck
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: make-lint-install
|
||||
name: install linters
|
||||
entry: make lint-install
|
||||
language: system
|
||||
pass_filenames: false
|
||||
|
||||
- id: make-lint
|
||||
name: run linters
|
||||
entry: make lint
|
||||
language: system
|
||||
pass_filenames: false
|
||||
|
||||
- id: go-unit-tests
|
||||
name: go unit tests
|
||||
entry: make test
|
||||
pass_filenames: false
|
||||
types: [go]
|
||||
language: system
|
559
CHANGELOG.md
559
CHANGELOG.md
|
@ -4,160 +4,423 @@ This document outlines major changes between releases.
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
- Fix marshaling errors in `DeleteObjects` method (#222)
|
||||
- Fix status code in GET/HEAD delete marker (#226)
|
||||
- Fix `NextVersionIDMarker` in `list-object-versions` (#248)
|
||||
- Fix possibility of panic during SIGHUP (#288)
|
||||
- Fix flaky `TestErrorTimeoutChecking` (`make test` sometimes failed) (#290)
|
||||
- Fix user owner ID in billing metrics (#321)
|
||||
- Fix HTTP/2 requests (#341)
|
||||
- Fix Decoder.CharsetReader is nil (#379)
|
||||
|
||||
### Added
|
||||
- Add new `frostfs.buffer_max_size_for_put` config param and sync TZ hash for PUT operations (#197)
|
||||
- Add `X-Amz-Version-Id` header after complete multipart upload (#227)
|
||||
- Add handling of `X-Amz-Copy-Source-Server-Side-Encryption-Customer-*` headers during copy (#217)
|
||||
- Add new `logger.destination` config param (#236)
|
||||
- Add `X-Amz-Content-Sha256` header validation (#218)
|
||||
- Support frostfsid contract. See `frostfsid` config section (#260)
|
||||
- Support per namespace placement policies configuration (see `namespaces.config` config param) (#266)
|
||||
- Support control api to manage policies. See `control` config section (#258)
|
||||
- Add `namespace` label to billing metrics (#271)
|
||||
- Support policy-engine (#257)
|
||||
- Support `policy` contract (#259)
|
||||
- Support `proxy` contract (#287)
|
||||
- Authmate: support custom attributes (#292)
|
||||
- Add new `reconnect_interval` config param (#291)
|
||||
- Support `GetBucketPolicyStatus` (#301)
|
||||
- Add FrostfsID cache (#269)
|
||||
- Add new `source_ip_header` config param (#371)
|
||||
- Return container name in `head-bucket` response (TrueCloudLab#18)
|
||||
- Billing metrics (TrueCloudLab#5)
|
||||
- Multiple configs support (TrueCloudLab#21)
|
||||
- Bucket name resolving policy (TrueCloudLab#25)
|
||||
- Support string `Action` and `Resource` fields in `bucketPolicy.Statement` (TrueCloudLab#32)
|
||||
|
||||
### Changed
|
||||
- Generalise config param `use_default_xmlns_for_complete_multipart` to `use_default_xmlns` so that use default xmlns for all requests (#221)
|
||||
- Set server IdleTimeout and ReadHeaderTimeout to `30s` and allow to configure them (#220)
|
||||
- Return `ETag` value in quotes (#219)
|
||||
- Use tombstone when delete multipart upload (#275)
|
||||
- Support new parameter `cache.accessbox.removing_check_interval` (#305)
|
||||
- Use APE rules instead of eACL in container creation (#306)
|
||||
|
||||
### Removed
|
||||
- Drop sending whitespace characters during complete multipart upload and related config param `kludge.complete_multipart_keepalive` (#227)
|
||||
|
||||
## [0.28.1] - 2024-01-24
|
||||
|
||||
### Added
|
||||
- MD5 hash as ETag and response header (#205)
|
||||
- Tree pool traversal limit (#262)
|
||||
|
||||
### Updating from 0.28.0
|
||||
|
||||
See new `features.md5.enabled` and `frostfs.tree_pool_max_attempts` config
|
||||
parameters.
|
||||
|
||||
## [0.28.0] - Academy of Sciences - 2023-12-07
|
||||
|
||||
### Fixed
|
||||
- Handle negative `Content-Length` on put (#125)
|
||||
- Use `DisableURIPathEscaping` to presign urls (#125)
|
||||
- Use specific s3 errors instead of `InternalError` where possible (#143)
|
||||
- `grpc` schemas in tree configuration (#166)
|
||||
- Return appropriate 404 code when object missed in storage but there is in gate cache (#158)
|
||||
- Replace part on re-upload when use multipart upload (#176)
|
||||
- Fix goroutine leak on put object error (#178)
|
||||
- Fix parsing signed headers in presigned urls (#182)
|
||||
- Fix url escaping (#188)
|
||||
- Use correct keys in `list-multipart-uploads` response (#185)
|
||||
- Fix parsing `key-marker` for object list versions (#237)
|
||||
- `GetSubTree` failures (#179)
|
||||
- Unexpected EOF during multipart download (#210)
|
||||
- Produce clean version in debian build (#245)
|
||||
|
||||
### Added
|
||||
- Add `trace_id` value into log record when tracing is enabled (#142)
|
||||
- Add basic error types and exit codes to `frostfs-s3-authmate` (#152)
|
||||
- Add a metric with addresses of nodes of the same and highest priority that are currently healthy (#186)
|
||||
- Support dump metrics descriptions (#80)
|
||||
- Add `copies_numbers` section to `placement_policy` in config file and support vectors of copies numbers (#70, #101)
|
||||
- Support impersonate bearer token (#81, #105)
|
||||
- Reload default and custom copies numbers on SIGHUP (#104)
|
||||
- Tracing support (#84, #140)
|
||||
- Return bearer token in `s3-authmate obtain-secret` result (#132)
|
||||
- Support multiple version credentials using GSet (#135)
|
||||
- Implement chunk uploading (#106)
|
||||
- Add new `kludge.bypass_content_encoding_check_in_chunks` config param (#146)
|
||||
- Add new `frostfs.client_cut` config param (#192)
|
||||
- Add selection of the node of the latest version of the object (#231)
|
||||
- Soft memory limit with `runtime.soft_memory_limit` (#196)
|
||||
- `server_health` metric for every S3 endpoint status (#199)
|
||||
|
||||
### Changed
|
||||
- Update prometheus to v1.15.0 (#94)
|
||||
- Update go version to go1.19 (#118)
|
||||
- Remove object from tree and reset its cache on object deletion when it is already removed from storage (#78)
|
||||
- Finish rebranding (#2)
|
||||
- Timeout errors has code 504 now (#103)
|
||||
- Use request scope logger (#111)
|
||||
- Add `s3-authmate update-secret` command (#131)
|
||||
- Use default registerer for app metrics (#155)
|
||||
- Use chi router instead of archived gorlilla/mux (#149, #174, #188)
|
||||
- Complete multipart upload doesn't unnecessary copy now. Thus, the total time of multipart upload was reduced by 2 times (#63)
|
||||
- Use gate key to form object owner (#175)
|
||||
- Apply placement policies and copies if there is at least one valid value (#168)
|
||||
- `statistic_tx_bytes_total` and `statistic_rx_bytes_total` metric to `statistic_bytes_total` metric with `direction` label (#153)
|
||||
- Refactor of context-stored data receivers (#137)
|
||||
- Refactor fetch/parse config parameters functions (#117)
|
||||
- Move all log messages to constants (#96)
|
||||
- Allow zero value of `part-number-marker` (#207)
|
||||
- Clean tag node in the tree service instead of removal (#233)
|
||||
|
||||
### Removed
|
||||
- Drop `tree.service` param (now endpoints from `peers` section are used) (#133)
|
||||
|
||||
## [0.27.0] - Karpinsky - 2023-07-12
|
||||
|
||||
This is a first FrostFS S3 Gateway release named after
|
||||
[Karpinsky glacier](https://en.wikipedia.org/wiki/Karpinsky_Glacier).
|
||||
|
||||
### Fixed
|
||||
- Using multiple servers require only one healthy (#12)
|
||||
- Renew token before it expires (#20)
|
||||
- Add generated deb builder files to .gitignore, and fix typo (#28)
|
||||
- Get empty bucket CORS from frostfs (#36)
|
||||
- Don't count pool error on client abort (#35)
|
||||
- Handle request cancelling (#69)
|
||||
- Clean up List and Name caches when object is missing in Tree service (#57)
|
||||
- Don't create unnecessary delete-markers (#83)
|
||||
- `Too many pings` error (#145)
|
||||
|
||||
### Added
|
||||
- Billing metrics (#5, #26, #29)
|
||||
- Return container name in `head-bucket` response (#18)
|
||||
- Multiple configs support (#21)
|
||||
- Bucket name resolving policy (#25)
|
||||
- Support string `Action` and `Resource` fields in `bucketPolicy.Statement` (#32)
|
||||
- Add new `kludge.use_default_xmlns_for_complete_multipart` config param (#40)
|
||||
- Return `X-Owner-Id` in `head-bucket` response (#79)
|
||||
- Support multiple tree service endpoints (#74, #110, #114)
|
||||
|
||||
### Changed
|
||||
- Repository rebranding (#1)
|
||||
- Update neo-go to v0.101.0 (#14)
|
||||
- Update viper to v1.15.0 (#14)
|
||||
- Update go version to go1.18 (#16)
|
||||
- Return error on invalid LocationConstraint (#23)
|
||||
- Limit number of objects to delete at one time (#37)
|
||||
- CompleteMultipartUpload handler now sends whitespace characters to keep alive client's connection (#60)
|
||||
- Support new system attributes (#64)
|
||||
- Abstract network communication in TreeClient (#59, #75)
|
||||
- Changed values for `frostfs_s3_gw_state_health` metric (#91)
|
||||
- Using multiple servers require only one healthy (TrueCloudLab#12)
|
||||
- Update go version to go1.18 (TrueCloudLab#16)
|
||||
- Return error on invalid LocationConstraint (TrueCloudLab#23)
|
||||
- Place billing metrics to separate url path (TrueCloudLab#26)
|
||||
- Add generated deb builder files to .gitignore, and fix typo (TrueCloudLab#28)
|
||||
|
||||
## [0.26.0] - 2022-12-28
|
||||
|
||||
### Added
|
||||
- Use client time as `now` in some requests (#726)
|
||||
- Reload policies on SIGHUP (#747)
|
||||
- Authmate flags for pool timeouts (#760)
|
||||
- Multiple server listeners (#742)
|
||||
|
||||
### Changed
|
||||
- Placement policy configuration (#568)
|
||||
- Improved debug logging of CID and OID values (#754)
|
||||
|
||||
### Removed
|
||||
- Deprecated linters (#755)
|
||||
|
||||
### Updating from v0.25.1
|
||||
New config parameters were added. And old one `defaul_policy` were changed.
|
||||
```yaml
|
||||
placement_policy:
|
||||
default: "REP 3"
|
||||
region_mapping: /path/to/container/policies.json
|
||||
```
|
||||
|
||||
Make sure you update the config accordingly:
|
||||
If you configure application using environment variables change:
|
||||
* `S3_GW_DEFAULT_POLICY` -> `S3_GW_PLACEMENT_POLICY_DEFAULT_POLICY`
|
||||
* `S3_GW_LISTEN_ADDRESS` -> `S3_GW_SERVER_0_ADDRESS`
|
||||
* `S3_GW_TLS_CERT_FILE` -> `S3_GW_SERVER_0_TLS_CERT_FILE` (and set `S3_GW_SERVER_0_TLS_ENABLED=true`)
|
||||
* `S3_GW_TLS_KEY_FILE` -> `S3_GW_SERVER_0_TLS_KEY_FILE` (and set `S3_GW_SERVER_0_TLS_ENABLED=true`)
|
||||
|
||||
If you configure application using `.yaml` file change:
|
||||
* `defaul_policy` -> `placement_policy.default`
|
||||
* `listen_address` -> `server.0.address`
|
||||
* `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`)
|
||||
* `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`)
|
||||
|
||||
## [0.25.1] - 2022-10-30
|
||||
|
||||
### Fixed
|
||||
- Empty bucket policy (#740)
|
||||
- Big object removal (#749)
|
||||
- Checksum panic (#741)
|
||||
|
||||
### Added
|
||||
- Debian packaging (#737)
|
||||
- Timeout for individual operations in streaming RPC (#750)
|
||||
|
||||
## [0.25.0] - 2022-10-31
|
||||
|
||||
### Fixed
|
||||
- Legal hold object lock enabling (#709)
|
||||
- Errors at object locking (#719)
|
||||
- Unrestricted access to not owned objects via cache (#713)
|
||||
- Check tree service health (#699)
|
||||
- Bucket names in listing (#733)
|
||||
|
||||
### Added
|
||||
- Config reloading on SIGHUP (#702, #715, #716)
|
||||
- Stop pool dial on SIGINT (#712)
|
||||
|
||||
### Changed
|
||||
- GitHub actions update (#710)
|
||||
- Makefile help (#725)
|
||||
- Optimized object tags setting (#669)
|
||||
- Improved logging (#728)
|
||||
- Unified unit test names (#617)
|
||||
- Improved docs (#732)
|
||||
|
||||
### Removed
|
||||
- Unused cache methods (#650)
|
||||
|
||||
### Updating from v0.24.0
|
||||
New config parameters were added. Make sure the default parameters are appropriate for you.
|
||||
|
||||
```yaml
|
||||
cache:
|
||||
accesscontrol:
|
||||
lifetime: 1m
|
||||
size: 100000
|
||||
```
|
||||
|
||||
## [0.24.0] - 2022-09-14
|
||||
|
||||
### Added
|
||||
- Exposure of pool metrics (#615, #680)
|
||||
- Configuration of `set_copies_number` (#634, #637)
|
||||
- Configuration of list of allowed `AccessKeyID` prefixes (#674)
|
||||
- Tagging directive for `CopyObject` (#666, #683)
|
||||
- Customer encryption (#595)
|
||||
- `CopiesNumber` configuration (#634, #637)
|
||||
|
||||
### Changed
|
||||
- Improved wallet configuration via `.yaml` config and environment variables (#607)
|
||||
- Update go version for build to 1.19 (#694, #705)
|
||||
- Update version calculation (#653, #697)
|
||||
- Optimized lock creation (#692)
|
||||
- Update way to configure `listen_domains` (#667)
|
||||
- Use `FilePath` instead of `FileName` for object keys (#657)
|
||||
- Optimize listing (#625, #616)
|
||||
|
||||
### Removed
|
||||
- Drop any object search logic (#545)
|
||||
|
||||
### Fixed
|
||||
- Responses to `GetObject` and `HeadObject`: removed redundant `VersionID` (#577, #682)
|
||||
- Replacement of object tagging in case of overwriting of an object (#645)
|
||||
- Using tags cache with empty `versionId` (#643)
|
||||
- Fix panic on go1.19 (#678)
|
||||
- Fix panic on invalid versioning status (#660)
|
||||
- Fix panic on missing decrypt reader (#704)
|
||||
- Using multipart uploads with `/` in name (#671)
|
||||
- Don't update settings cache when request fails (#661)
|
||||
- Fix handling `X-Amz-Copy-Source` header (#672)
|
||||
- ACL related problems (#676, #606)
|
||||
- Using `ContinuationToken` for "directories" (#684)
|
||||
- Fix `connection was closed` error (#656)
|
||||
- Fix listing for nested objects (#624)
|
||||
- Fix anon requests to tree service (#504, #505)
|
||||
|
||||
### Updating from v0.23.0
|
||||
Make sure your configuration is valid:
|
||||
|
||||
If you configure application using environment variables change:
|
||||
* `S3_GW_WALLET` -> `S3_GW_WALLET_PATH`
|
||||
* `S3_GW_ADDRESS` -> `S3_GW_WALLET_ADDRESS`
|
||||
* `S3_GW_LISTEN_DOMAINS_N` -> `S3_GW_LISTEN_DOMAINS` (use it as array variable)
|
||||
|
||||
If you configure application using `.yaml` file change:
|
||||
* `wallet` -> `wallet.path`
|
||||
* `address` -> `wallet.address`
|
||||
* `listen_domains.n` -> `listen_domains` (use it as array param)
|
||||
|
||||
|
||||
## [0.23.0] - 2022-08-01
|
||||
|
||||
### Fixed
|
||||
- System metadata are filtered now (#619)
|
||||
- List objects in corner cases (#612, #627)
|
||||
- Correct removal of a deleted object (#610)
|
||||
- Bucket creation could lead to "no healthy client" error (#636)
|
||||
|
||||
### Added
|
||||
- New param to configure pool error threshold (#633)
|
||||
|
||||
### Changed
|
||||
- Pprof and prometheus metrics configuration (#591)
|
||||
- Don't set sticky bit in authmate container (#540)
|
||||
- Updated compatibility table (#638)
|
||||
- Rely on string sanitizing from zap (#498)
|
||||
|
||||
### Updating from v0.22.0
|
||||
1. To enable pprof use `pprof.enabled` instead of `pprof` in config.
|
||||
To enable prometheus metrics use `prometheus.enabled` instead of `metrics` in config.
|
||||
If you are using the command line flags you can skip this step.
|
||||
|
||||
## [0.22.0] - 2022-07-25
|
||||
|
||||
Tree service support
|
||||
|
||||
### Fixed
|
||||
- Error logging (#450)
|
||||
- Default bucket location constraint (#463)
|
||||
- Suspended versioning status (#462)
|
||||
- CodeQL warnings (#489, #522, #539)
|
||||
- Bearer token behaviour with non-owned buckets (#459)
|
||||
- ACL issues (#495, #553, #571, #573, #574, #580)
|
||||
- Authmate policy parsing (#558)
|
||||
|
||||
### Added
|
||||
- Public key output in authmate issue-secret command (#482)
|
||||
- Support of conditional headers (#484)
|
||||
- Cache type cast error logging (#465)
|
||||
- `docker/*` target in Makefile (#471)
|
||||
- Pre signed requests (#529)
|
||||
- Tagging and ACL notifications (#361)
|
||||
- AWSv4 signer package to improve compatibility with S3 clients (#528)
|
||||
- Extension mimetype detector (#289)
|
||||
- Default params documentation (#592)
|
||||
- Health metric (#600)
|
||||
- Parallel object listing (#525)
|
||||
- Tree service (see commit links from #609)
|
||||
|
||||
### Changed
|
||||
- Reduce number of network requests (#439, #441)
|
||||
- Renamed authmate to s3-authmate (#518)
|
||||
- Version output (#578)
|
||||
- Improved error messages (#539)
|
||||
|
||||
### Removed
|
||||
- `layer/neofs` package (#438)
|
||||
|
||||
## [0.21.1] - 2022-05-16
|
||||
|
||||
### Changed
|
||||
- Update go version to go1.17 (#427)
|
||||
- Set homomorphic hashing disable attribute in container if required (#435)
|
||||
|
||||
## [0.21.0] - 2022-05-13
|
||||
|
||||
### Added
|
||||
- Support of get-object-attributes (#430)
|
||||
|
||||
### Fixed
|
||||
- Reduced time of bucket creation (#426)
|
||||
- Bucket removal (#428)
|
||||
- Obtainment of ETag value (#431)
|
||||
|
||||
### Changed
|
||||
- Authmate doesn't parse session context anymore, now it accepts application defined
|
||||
flexible structure with container ID in human-readable format (#428)
|
||||
|
||||
## [0.20.0] - 2022-04-29
|
||||
|
||||
### Added
|
||||
- Support of object locking (#195)
|
||||
- Support of basic notifications (#357, #358, #359)
|
||||
|
||||
### Changed
|
||||
- Logger behavior: now it writes to stderr instead of stdout, app name and
|
||||
version are always presented and fixed, all user options except of `level` are
|
||||
dropped (#380)
|
||||
- Improved docs, added config examples (#396, #398)
|
||||
- Updated NeoFS SDK (#365, #409)
|
||||
|
||||
### Fixed
|
||||
- Added check of `SetEACL` tokens before processing of requests (#347)
|
||||
- Authmate: returned lost session tokens when a parameter `--session-token` is
|
||||
omitted (#387)
|
||||
- Error when a bucket hasn't a settings file (#389)
|
||||
- Response to a request to delete not existing object (#392)
|
||||
- Replaced gate key in ACL Grantee by key of bearer token issuer (#395)
|
||||
- Missing attach of bearer token to requests to put system object (#399)
|
||||
- Deletion of system object while CompleteMultipartUpload (#400)
|
||||
- Improved English in docs and comments (#405)
|
||||
- Authmate: reconsidered default bearer token rules (#406)
|
||||
|
||||
## [0.19.0] - 2022-03-16
|
||||
|
||||
### Added
|
||||
- Authmate: support placement policy overriding (#343, #364)
|
||||
- Managing bucket notification configuration (#340)
|
||||
- Unit tests in go1.17 (#265)
|
||||
- NATS settings in application config (#341)
|
||||
- Support `Expires` and `Cache-Control` headers (#312)
|
||||
- Support `%` as delimiter (#313)
|
||||
- Support `null` version deletion (#319)
|
||||
- Bucket name resolving order (#285)
|
||||
- Authmate: added `timeout` flag (#290)
|
||||
- MinIO results in s3 compatibility tables (#304)
|
||||
- Support overriding response headers (#310)
|
||||
|
||||
### Changed
|
||||
- Authmate: check parameters before container creation (#372)
|
||||
- Unify cache invalidation on deletion (#368)
|
||||
- Updated NeoFS SDK to v1.0.0-rc.3 (#297, #333, #346, #376)
|
||||
- Authmate: changed session token rules handling (#329, #336, #338, #352)
|
||||
- Changed status code for some failed requests (#308)
|
||||
- GetBucketLocation returns policy name used at bucket creation (#301)
|
||||
|
||||
### Fixed
|
||||
- Waiting for bucket to be deleted (#366)
|
||||
- Authmate: changed error message for session context building (#348)
|
||||
- Authmate: fixed access key parsing in `obtain-secret` command (#295)
|
||||
- Distinguishing `BucketAlreadyExists` errors (#354)
|
||||
- Incorrect panic if handler not found (#305)
|
||||
- Authmate: use container friendly name as system name (#299, #324)
|
||||
- Use UTC `Last-Modified` timestamps (#331)
|
||||
- Don't return object system metadata (#307)
|
||||
- Handling empty post policy (#306)
|
||||
- Use `X-Amz-Verion-Id` in `CompleteMulipartUpload` (#318)
|
||||
|
||||
### Removed
|
||||
- Drop MinIO related errors (#316)
|
||||
|
||||
## [0.18.0] - 2021-12-16
|
||||
|
||||
### Added
|
||||
- Support for MultipartUpload (#186, #187)
|
||||
- CORS support (#217)
|
||||
- Authmate supports setting of tokens lifetime in a more convenient format (duration) (#258)
|
||||
- Generation of a random key for `--no-sign-request` (#276)
|
||||
|
||||
### Changed
|
||||
- Bucket name resolving mechanism from listing owner's containers to using DNS (#219)
|
||||
|
||||
### Removed
|
||||
- Deprecated golint, replaced by revive (#272)
|
||||
|
||||
## 0.17.0 (24 Sep 2021)
|
||||
With this release we introduce [ceph-based](https://github.com/ceph/s3-tests) S3 compatibility results.
|
||||
|
||||
### Added
|
||||
* Versioning support (#122, #242, #263)
|
||||
* Ceph S3 compatibility results (#150, #249, #266)
|
||||
* Handling `X-Amz-Expected-Bucket-Owner` header (#216)
|
||||
* `X-Container-Id` header for `HeadBucket` response (#220)
|
||||
* Basic ACL support (#49, #213)
|
||||
* Caching (#179, #206, #231, #236, #253)
|
||||
* Metadata directive when copying (#191)
|
||||
* Bucket name checking (189)
|
||||
* Continuation token support (#112, #154, #180)
|
||||
* Mapping `LocationConstraint` to `PlacementPolicy` (#89)
|
||||
* Tagging support (#196)
|
||||
* POST uploading support (#190)
|
||||
* Delete marker support (#248)
|
||||
* Expiration for access box (#255)
|
||||
* AWS CLI credential generating by authmate (#241)
|
||||
|
||||
### Changed
|
||||
* Default placement policy is now configurable (#218)
|
||||
* README is split into different files (#210)
|
||||
* Unified error handling (#89, #149, #184)
|
||||
* Authmate issue-secret response contains container id (#163)
|
||||
* Removed "github.com/nspcc-dev/neofs-node" dependency (#234)
|
||||
* Removed GitHub workflow of image publishing (#243)
|
||||
* Changed license to AGPLv3 (#264)
|
||||
|
||||
### Fixed
|
||||
* ListObjects results are now the same for different users (#230)
|
||||
* Error response for invalid authentication header is now correct (#199)
|
||||
* Saving object metadata (#198)
|
||||
* Range header handling (#194)
|
||||
* Correct status codes (#118, #262)
|
||||
* HeadObject for "directories" (#160)
|
||||
* Fetch-owner parameter support (#159)
|
||||
|
||||
## 0.16.0 (16 Jul 2021)
|
||||
|
||||
With this release we publish S3 gateway source code. It includes various S3
|
||||
compatibility improvements, support of bucket management, unified secp256r1
|
||||
cryptography with NEP-6 wallet support.
|
||||
|
||||
### Fixed
|
||||
* Allowed no-sign request (#65)
|
||||
* Bearer token attached to all requests (#84)
|
||||
* Time format in responses (#133)
|
||||
* Max-keys checked in ListObjects (#135)
|
||||
* Lost metadat in the objects (#131)
|
||||
* Unique bucket name check (#125)
|
||||
|
||||
### Added
|
||||
* Bucket management operations (#47, #72)
|
||||
* Node-specific owner IDs in bearer tokens (#83)
|
||||
* AWS CLI usage section in README (#77)
|
||||
* List object paging (#97)
|
||||
* Lifetime for the tokens in auth-mate (#108)
|
||||
* Support of range in GetObject request (#96)
|
||||
* Support of NEP-6 wallets instead of binary encoded keys (#92)
|
||||
* Support of JSON encoded rules in auth-mate (#71)
|
||||
* Support of delimiters in ListObjects (#98)
|
||||
* Support of object ETag (#93)
|
||||
* Support of time-based conditional CopyObject and GetObject (#94)
|
||||
|
||||
### Changed
|
||||
* Accesskey format: now `0` used as a delimiter between container ID and object
|
||||
ID instead of `_` (#164)
|
||||
* Accessbox is encoded in protobuf format (#48)
|
||||
* Authentication uses secp256r1 instead of ed25519 (#75)
|
||||
* Improved integration with NeoFS SDK and NeoFS API Go (#78, #88)
|
||||
* Optimized object put execution (#155)
|
||||
|
||||
### Removed
|
||||
* GRPC keepalive options (#73)
|
||||
|
||||
## 0.15.0 (10 Jun 2021)
|
||||
|
||||
This release brings S3 gateway to the current state of NeoFS and fixes some
|
||||
bugs, no new significant features introduced (other than moving here already
|
||||
existing authmate component).
|
||||
|
||||
New features:
|
||||
* authmate was moved into this repository and is now built along with the
|
||||
gateway itself (#46)
|
||||
|
||||
Behavior changes:
|
||||
* neofs-s3-gate was renamed to neofs-s3-gw (#50)
|
||||
|
||||
Improvements:
|
||||
* better Makefile (#43, #45, #55)
|
||||
* stricter linters (#45)
|
||||
* removed non-standard errors package from dependencies (#54)
|
||||
* refactoring, reusing new sdk-go component (#60, #62, #63)
|
||||
* updated neofs-api-go for compatibility with current NeoFS node 0.21.0 (#60, #68)
|
||||
* extended README (#67, #76)
|
||||
|
||||
Bugs fixed:
|
||||
* wrong (as per AWS specification) access key ID generated (#64)
|
||||
|
||||
## Older versions
|
||||
|
||||
This project is a fork of [NeoFS S3 Gateway](https://github.com/nspcc-dev/neofs-s3-gw) from version v0.26.0.
|
||||
To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs-s3-gw/blob/master/CHANGELOG.md.
|
||||
Please refer to [Github
|
||||
releases](https://github.com/nspcc-dev/neofs-s3-gw/releases/) for older
|
||||
releases.
|
||||
|
||||
[0.27.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/b2148cc3...v0.27.0
|
||||
[0.28.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.27.0...v0.28.0
|
||||
[0.28.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.28.0...v0.28.1
|
||||
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.28.1...master
|
||||
[0.18.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.17.0...v0.18.0
|
||||
[0.19.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.18.0...v0.19.0
|
||||
[0.20.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.19.0...v0.20.0
|
||||
[0.21.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.20.0...v0.21.0
|
||||
[0.21.1]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.21.0...v0.21.1
|
||||
[0.22.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.21.1...v0.22.0
|
||||
[0.23.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.22.0...v0.23.0
|
||||
[0.24.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.23.0...v0.24.0
|
||||
[0.25.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.24.0...v0.25.0
|
||||
[Unreleased]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.25.0...master
|
||||
|
|
|
@ -3,8 +3,8 @@
|
|||
First, thank you for contributing! We love and encourage pull requests from
|
||||
everyone. Please follow the guidelines:
|
||||
|
||||
- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/issues) and
|
||||
[pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pulls) for existing
|
||||
- Check the open [issues](https://github.com/TrueCloudLab/frostfs-s3-gw/issues) and
|
||||
[pull requests](https://github.com/TrueCloudLab/frostfs-s3-gw/pulls) for existing
|
||||
discussions.
|
||||
|
||||
- Open an issue first, to discuss a new feature or enhancement.
|
||||
|
@ -27,20 +27,20 @@ Start by forking the `frostfs-s3-gw` repository, make changes in a branch and th
|
|||
send a pull request. We encourage pull requests to discuss code changes. Here
|
||||
are the steps in details:
|
||||
|
||||
### Set up your git repository
|
||||
### Set up your GitHub Repository
|
||||
Fork [FrostFS S3 Gateway
|
||||
upstream](https://git.frostfs.info/repo/fork/15) source repository
|
||||
upstream](https://github.com/TrueCloudLab/frostfs-s3-gw/fork) source repository
|
||||
to your own personal repository. Copy the URL of your fork (you will need it for
|
||||
the `git clone` command below).
|
||||
|
||||
```sh
|
||||
$ git clone https://git.frostfs.info/<username>/frostfs-s3-gw.git
|
||||
$ git clone https://github.com/TrueCloudLab/frostfs-s3-gw
|
||||
```
|
||||
|
||||
### Set up git remote as ``upstream``
|
||||
```sh
|
||||
$ cd frostfs-s3-gw
|
||||
$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw.git
|
||||
$ git remote add upstream https://github.com/TrueCloudLab/frostfs-s3-gw
|
||||
$ git fetch upstream
|
||||
$ git merge upstream/master
|
||||
...
|
||||
|
@ -90,8 +90,8 @@ $ git push origin feature/123-something_awesome
|
|||
```
|
||||
|
||||
### Create a Pull Request
|
||||
Pull requests can be created via Forgejo. Refer to [this
|
||||
document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for
|
||||
Pull requests can be created via GitHub. Refer to [this
|
||||
document](https://help.github.com/articles/creating-a-pull-request/) for
|
||||
detailed steps on how to create a pull request. After a Pull Request gets peer
|
||||
reviewed and approved, it will be merged.
|
||||
|
||||
|
|
|
@ -16,4 +16,4 @@ In chronological order:
|
|||
- Elizaveta Chichindaeva
|
||||
- Stanislav Bogatyrev
|
||||
- Anastasia Prasolova
|
||||
- Leonard Liubich
|
||||
- Leonard Liubich
|
60
Makefile
Executable file → Normal file
60
Makefile
Executable file → Normal file
|
@ -3,13 +3,10 @@
|
|||
# Common variables
|
||||
REPO ?= $(shell go list -m)
|
||||
VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
|
||||
GO_VERSION ?= 1.20
|
||||
LINT_VERSION ?= 1.56.1
|
||||
TRUECLOUDLAB_LINT_VERSION ?= 0.0.5
|
||||
GO_VERSION ?= 1.19
|
||||
LINT_VERSION ?= 1.49.0
|
||||
BINDIR = bin
|
||||
|
||||
METRICS_DUMP_OUT ?= ./metrics-dump.json
|
||||
|
||||
# Binaries to build
|
||||
CMDS = $(addprefix frostfs-, $(notdir $(wildcard cmd/*)))
|
||||
BINS = $(addprefix $(BINDIR)/, $(CMDS))
|
||||
|
@ -19,23 +16,19 @@ REPO_BASENAME = $(shell basename `go list -m`)
|
|||
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
|
||||
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||
|
||||
OUTPUT_LINT_DIR ?= $(shell pwd)/bin
|
||||
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
|
||||
TMP_DIR := .cache
|
||||
|
||||
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
|
||||
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint version clean protoc
|
||||
|
||||
# .deb package versioning
|
||||
OS_RELEASE = $(shell lsb_release -cs)
|
||||
PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
|
||||
sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \
|
||||
sed "s/-/~/")-${OS_RELEASE}
|
||||
.PHONY: debpackage debclean
|
||||
.PHONY: debpackage debclean
|
||||
|
||||
# Make all binaries
|
||||
all: $(BINS)
|
||||
|
||||
$(BINS): $(BINDIR) dep
|
||||
$(BINS): sync-tree $(BINDIR) dep
|
||||
@echo "⇒ Build $@"
|
||||
CGO_ENABLED=0 \
|
||||
go build -v -trimpath \
|
||||
|
@ -46,6 +39,10 @@ $(BINDIR):
|
|||
@echo "⇒ Ensure dir: $@"
|
||||
@mkdir -p $@
|
||||
|
||||
# Synchronize tree service
|
||||
sync-tree:
|
||||
@./syncTree.sh
|
||||
|
||||
# Pull go dependencies
|
||||
dep:
|
||||
@printf "⇒ Download requirements: "
|
||||
|
@ -106,23 +103,9 @@ dirty-image:
|
|||
-f .docker/Dockerfile.dirty \
|
||||
-t $(HUB_IMAGE)-dirty:$(HUB_TAG) .
|
||||
|
||||
# Install linters
|
||||
lint-install:
|
||||
@mkdir -p $(TMP_DIR)
|
||||
@rm -rf $(TMP_DIR)/linters
|
||||
@git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters
|
||||
@@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR)
|
||||
@rm -rf $(TMP_DIR)/linters
|
||||
@rmdir $(TMP_DIR) 2>/dev/null || true
|
||||
@CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION)
|
||||
|
||||
# Run linters
|
||||
lint:
|
||||
@if [ ! -d "$(LINT_DIR)" ]; then \
|
||||
echo "Run make lint-install"; \
|
||||
exit 1; \
|
||||
fi
|
||||
$(LINT_DIR)/golangci-lint --timeout=5m run
|
||||
@golangci-lint --timeout=5m run
|
||||
|
||||
# Run linters in Docker
|
||||
docker/lint:
|
||||
|
@ -132,14 +115,6 @@ docker/lint:
|
|||
--env HOME=/src \
|
||||
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
|
||||
|
||||
# Activate pre-commit hooks
|
||||
pre-commit:
|
||||
pre-commit install -t pre-commit -t commit-msg
|
||||
|
||||
# Deactivate pre-commit hooks
|
||||
unpre-commit:
|
||||
pre-commit uninstall -t pre-commit -t commit-msg
|
||||
|
||||
# Show current version
|
||||
version:
|
||||
@echo $(VERSION)
|
||||
|
@ -151,16 +126,10 @@ clean:
|
|||
|
||||
# Generate code from .proto files
|
||||
protoc:
|
||||
# Install specific version for protobuf lib
|
||||
@GOBIN=$(abspath $(BINDIR)) go install -mod=mod -v git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/protogen
|
||||
@for f in `find . -type f -name '*.proto' -not -path './vendor/*'`; do \
|
||||
echo "⇒ Processing $$f "; \
|
||||
protoc \
|
||||
--go_out=paths=source_relative:. \
|
||||
--plugin=protoc-gen-go-frostfs=$(BINDIR)/protogen \
|
||||
--go-frostfs_out=. --go-frostfs_opt=paths=source_relative \
|
||||
--go-grpc_opt=require_unimplemented_servers=false \
|
||||
--go-grpc_out=. --go-grpc_opt=paths=source_relative $$f; \
|
||||
--go_out=paths=source_relative:. $$f; \
|
||||
done
|
||||
rm -rf vendor
|
||||
|
||||
|
@ -174,11 +143,6 @@ debpackage:
|
|||
dpkg-buildpackage --no-sign -b
|
||||
|
||||
debclean:
|
||||
dh clean
|
||||
|
||||
# Dump metrics (use METRICS_DUMP_OUT variable to override default out file './metrics-dump.json')
|
||||
.PHONY: dump-metrics
|
||||
dump-metrics:
|
||||
@go test ./metrics -run TestDescribeAll --tags=dump_metrics --out=$(abspath $(METRICS_DUMP_OUT))
|
||||
dh clean
|
||||
|
||||
include help.mk
|
||||
|
|
12
README.md
12
README.md
|
@ -6,9 +6,9 @@
|
|||
</p>
|
||||
|
||||
---
|
||||
[![Report](https://goreportcard.com/badge/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)
|
||||
![Release](https://img.shields.io/badge/dynamic/json.svg?label=release&url=https://git.frostfs.info/api/v1/repos/TrueCloudLab/frostfs-s3-gw/releases&query=$[0].tag_name&color=orange)
|
||||
![License](https://img.shields.io/badge/license-GPL--3.0-orange.svg)
|
||||
[![Report](https://goreportcard.com/badge/github.com/TrueCloudLab/frostfs-s3-gw)](https://goreportcard.com/report/github.com/TrueCloudLab/frostfs-s3-gw)
|
||||
![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/TrueCloudLab/frostfs-s3-gw?sort=semver)
|
||||
![License](https://img.shields.io/github/license/TrueCloudLab/frostfs-s3-gw.svg?style=popout)
|
||||
|
||||
# FrostFS S3 Gateway
|
||||
|
||||
|
@ -16,7 +16,7 @@ FrostFS S3 gateway provides API compatible with Amazon S3 cloud storage service.
|
|||
|
||||
## Installation
|
||||
|
||||
```go get -u git.frostfs.info/TrueCloudLab/frostfs-s3-gw```
|
||||
```go get -u github.com/TrueCloudLab/frostfs-s3-gw```
|
||||
|
||||
Or you can call `make` to build it from the cloned repository (the binary will
|
||||
end up in `bin/frostfs-s3-gw` with authmate helper in `bin/frostfs-s3-authmate`).
|
||||
|
@ -68,7 +68,7 @@ $ S3_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 \
|
|||
|
||||
## Domains
|
||||
|
||||
By default, s3-gw enable only `path-style access`.
|
||||
By default, s3-gw enable only `path-style access`.
|
||||
To be able to use both: `virtual-hosted-style` and `path-style` access you must configure `listen_domains`:
|
||||
|
||||
```shell
|
||||
|
@ -102,6 +102,6 @@ Also, you can configure domains using `.env` variables or `yaml` file.
|
|||
- [AWS S3 API compatibility](./docs/aws_s3_compat.md)
|
||||
- [AWS S3 Compatibility test results](./docs/s3_test_results.md)
|
||||
|
||||
## Credits
|
||||
## Credits
|
||||
|
||||
Please see [CREDITS](CREDITS.md) for details.
|
||||
|
|
2
VERSION
2
VERSION
|
@ -1 +1 @@
|
|||
v0.28.1
|
||||
v0.26.0
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
|
@ -13,13 +14,14 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
v4 "github.com/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
apiErrors "github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
)
|
||||
|
||||
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
|
||||
|
@ -29,15 +31,27 @@ var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(
|
|||
var postPolicyCredentialRegexp = regexp.MustCompile(`(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request`)
|
||||
|
||||
type (
|
||||
Center struct {
|
||||
// Center is a user authentication interface.
|
||||
Center interface {
|
||||
Authenticate(request *http.Request) (*Box, error)
|
||||
}
|
||||
|
||||
// Box contains access box and additional info.
|
||||
Box struct {
|
||||
AccessBox *accessbox.Box
|
||||
ClientTime time.Time
|
||||
}
|
||||
|
||||
center struct {
|
||||
reg *RegexpSubmatcher
|
||||
postReg *RegexpSubmatcher
|
||||
cli tokens.Credentials
|
||||
allowedAccessKeyIDPrefixes []string // empty slice means all access key ids are allowed
|
||||
}
|
||||
|
||||
//nolint:revive
|
||||
AuthHeader struct {
|
||||
prs int
|
||||
|
||||
authHeader struct {
|
||||
AccessKeyID string
|
||||
Service string
|
||||
Region string
|
||||
|
@ -60,51 +74,47 @@ const (
|
|||
AmzSignedHeaders = "X-Amz-SignedHeaders"
|
||||
AmzExpires = "X-Amz-Expires"
|
||||
AmzDate = "X-Amz-Date"
|
||||
AmzContentSHA256 = "X-Amz-Content-Sha256"
|
||||
AuthorizationHdr = "Authorization"
|
||||
ContentTypeHdr = "Content-Type"
|
||||
|
||||
UnsignedPayload = "UNSIGNED-PAYLOAD"
|
||||
StreamingUnsignedPayloadTrailer = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
|
||||
StreamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||
StreamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
|
||||
StreamingContentECDSASHA256 = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
|
||||
StreamingContentECDSASHA256Trailer = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
|
||||
)
|
||||
|
||||
var ContentSHA256HeaderStandardValue = map[string]struct{}{
|
||||
UnsignedPayload: {},
|
||||
StreamingUnsignedPayloadTrailer: {},
|
||||
StreamingContentSHA256: {},
|
||||
StreamingContentSHA256Trailer: {},
|
||||
StreamingContentECDSASHA256: {},
|
||||
StreamingContentECDSASHA256Trailer: {},
|
||||
// ErrNoAuthorizationHeader is returned for unauthenticated requests.
|
||||
var ErrNoAuthorizationHeader = errors.New("no authorization header")
|
||||
|
||||
func (p prs) Read(_ []byte) (n int, err error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (p prs) Seek(_ int64, _ int) (int64, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
var _ io.ReadSeeker = prs(0)
|
||||
|
||||
// New creates an instance of AuthCenter.
|
||||
func New(creds tokens.Credentials, prefixes []string) *Center {
|
||||
return &Center{
|
||||
cli: creds,
|
||||
func New(frostFS tokens.FrostFS, key *keys.PrivateKey, prefixes []string, config *cache.Config) Center {
|
||||
return ¢er{
|
||||
cli: tokens.New(frostFS, key, config),
|
||||
reg: NewRegexpMatcher(authorizationFieldRegexp),
|
||||
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
||||
allowedAccessKeyIDPrefixes: prefixes,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Center) parseAuthHeader(header string) (*AuthHeader, error) {
|
||||
func (c *center) parseAuthHeader(header string) (*authHeader, error) {
|
||||
submatches := c.reg.GetSubmatches(header)
|
||||
if len(submatches) != authHeaderPartsNum {
|
||||
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrAuthorizationHeaderMalformed), header)
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrAuthorizationHeaderMalformed)
|
||||
}
|
||||
|
||||
accessKey := strings.Split(submatches["access_key_id"], "0")
|
||||
if len(accessKey) != accessKeyPartsNum {
|
||||
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrInvalidAccessKeyID), accessKey)
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidAccessKeyID)
|
||||
}
|
||||
|
||||
signedFields := strings.Split(submatches["signed_header_fields"], ";")
|
||||
|
||||
return &AuthHeader{
|
||||
return &authHeader{
|
||||
AccessKeyID: submatches["access_key_id"],
|
||||
Service: submatches["service"],
|
||||
Region: submatches["region"],
|
||||
|
@ -114,24 +124,18 @@ func (c *Center) parseAuthHeader(header string) (*AuthHeader, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
func getAddress(accessKeyID string) (oid.Address, error) {
|
||||
func (a *authHeader) getAddress() (oid.Address, error) {
|
||||
var addr oid.Address
|
||||
if err := addr.DecodeString(strings.ReplaceAll(accessKeyID, "0", "/")); err != nil {
|
||||
return addr, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrInvalidAccessKeyID), accessKeyID)
|
||||
if err := addr.DecodeString(strings.ReplaceAll(a.AccessKeyID, "0", "/")); err != nil {
|
||||
return addr, apiErrors.GetAPIError(apiErrors.ErrInvalidAccessKeyID)
|
||||
}
|
||||
|
||||
return addr, nil
|
||||
}
|
||||
|
||||
func IsStandardContentSHA256(key string) bool {
|
||||
_, ok := ContentSHA256HeaderStandardValue[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
|
||||
func (c *center) Authenticate(r *http.Request) (*Box, error) {
|
||||
var (
|
||||
err error
|
||||
authHdr *AuthHeader
|
||||
authHdr *authHeader
|
||||
signatureDateTimeStr string
|
||||
needClientTime bool
|
||||
)
|
||||
|
@ -142,12 +146,12 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
|
|||
if len(creds) != 5 || creds[4] != "aws4_request" {
|
||||
return nil, fmt.Errorf("bad X-Amz-Credential")
|
||||
}
|
||||
authHdr = &AuthHeader{
|
||||
authHdr = &authHeader{
|
||||
AccessKeyID: creds[0],
|
||||
Service: creds[3],
|
||||
Region: creds[2],
|
||||
SignatureV4: queryValues.Get(AmzSignature),
|
||||
SignedFields: strings.Split(queryValues.Get(AmzSignedHeaders), ";"),
|
||||
SignedFields: queryValues[AmzSignedHeaders],
|
||||
Date: creds[1],
|
||||
IsPresigned: true,
|
||||
}
|
||||
|
@ -162,7 +166,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
|
|||
if strings.HasPrefix(r.Header.Get(ContentTypeHdr), "multipart/form-data") {
|
||||
return c.checkFormData(r)
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %v", middleware.ErrNoAuthorizationHeader, authHeaderField)
|
||||
return nil, ErrNoAuthorizationHeader
|
||||
}
|
||||
authHdr, err = c.parseAuthHeader(authHeaderField[0])
|
||||
if err != nil {
|
||||
|
@ -177,22 +181,18 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
|
|||
return nil, fmt.Errorf("failed to parse x-amz-date header field: %w", err)
|
||||
}
|
||||
|
||||
if err = c.checkAccessKeyID(authHdr.AccessKeyID); err != nil {
|
||||
if err := c.checkAccessKeyID(authHdr.AccessKeyID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
addr, err := getAddress(authHdr.AccessKeyID)
|
||||
addr, err := authHdr.getAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
box, attrs, err := c.cli.GetBox(r.Context(), addr)
|
||||
box, err := c.cli.GetBox(r.Context(), addr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get box '%s': %w", addr, err)
|
||||
}
|
||||
|
||||
if err = checkFormatHashContentSHA256(r.Header.Get(AmzContentSHA256)); err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("get box: %w", err)
|
||||
}
|
||||
|
||||
clonedRequest := cloneRequest(r, authHdr)
|
||||
|
@ -200,15 +200,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
result := &middleware.Box{
|
||||
AccessBox: box,
|
||||
AuthHeaders: &middleware.AuthHeader{
|
||||
AccessKeyID: authHdr.AccessKeyID,
|
||||
Region: authHdr.Region,
|
||||
SignatureV4: authHdr.SignatureV4,
|
||||
},
|
||||
Attributes: attrs,
|
||||
}
|
||||
result := &Box{AccessBox: box}
|
||||
if needClientTime {
|
||||
result.ClientTime = signatureDateTime
|
||||
}
|
||||
|
@ -216,22 +208,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
|
|||
return result, nil
|
||||
}
|
||||
|
||||
func checkFormatHashContentSHA256(hash string) error {
|
||||
if !IsStandardContentSHA256(hash) {
|
||||
hashBinary, err := hex.DecodeString(hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: decode hash: %s: %s", apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch),
|
||||
hash, err.Error())
|
||||
}
|
||||
if len(hashBinary) != sha256.Size && len(hash) != 0 {
|
||||
return fmt.Errorf("%w: invalid hash size %d", apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch), len(hashBinary))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c Center) checkAccessKeyID(accessKeyID string) error {
|
||||
func (c center) checkAccessKeyID(accessKeyID string) error {
|
||||
if len(c.allowedAccessKeyIDPrefixes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
@ -242,12 +219,12 @@ func (c Center) checkAccessKeyID(accessKeyID string) error {
|
|||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("%w: accesskeyID prefix isn't allowed", apiErrors.GetAPIError(apiErrors.ErrAccessDenied))
|
||||
return apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
|
||||
}
|
||||
|
||||
func (c *Center) checkFormData(r *http.Request) (*middleware.Box, error) {
|
||||
func (c *center) checkFormData(r *http.Request) (*Box, error) {
|
||||
if err := r.ParseMultipartForm(maxFormSizeMemory); err != nil {
|
||||
return nil, fmt.Errorf("%w: parse multipart form with max size %d", apiErrors.GetAPIError(apiErrors.ErrInvalidArgument), maxFormSizeMemory)
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidArgument)
|
||||
}
|
||||
|
||||
if err := prepareForm(r.MultipartForm); err != nil {
|
||||
|
@ -256,13 +233,12 @@ func (c *Center) checkFormData(r *http.Request) (*middleware.Box, error) {
|
|||
|
||||
policy := MultipartFormValue(r, "policy")
|
||||
if policy == "" {
|
||||
return nil, fmt.Errorf("%w: missing policy", middleware.ErrNoAuthorizationHeader)
|
||||
return nil, ErrNoAuthorizationHeader
|
||||
}
|
||||
|
||||
creds := MultipartFormValue(r, "x-amz-credential")
|
||||
submatches := c.postReg.GetSubmatches(creds)
|
||||
submatches := c.postReg.GetSubmatches(MultipartFormValue(r, "x-amz-credential"))
|
||||
if len(submatches) != 4 {
|
||||
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrAuthorizationHeaderMalformed), creds)
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrAuthorizationHeaderMalformed)
|
||||
}
|
||||
|
||||
signatureDateTime, err := time.Parse("20060102T150405Z", MultipartFormValue(r, "x-amz-date"))
|
||||
|
@ -270,30 +246,28 @@ func (c *Center) checkFormData(r *http.Request) (*middleware.Box, error) {
|
|||
return nil, fmt.Errorf("failed to parse x-amz-date field: %w", err)
|
||||
}
|
||||
|
||||
addr, err := getAddress(submatches["access_key_id"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var addr oid.Address
|
||||
if err = addr.DecodeString(strings.ReplaceAll(submatches["access_key_id"], "0", "/")); err != nil {
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidAccessKeyID)
|
||||
}
|
||||
|
||||
box, attrs, err := c.cli.GetBox(r.Context(), addr)
|
||||
box, err := c.cli.GetBox(r.Context(), addr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get box '%s': %w", addr, err)
|
||||
return nil, fmt.Errorf("get box: %w", err)
|
||||
}
|
||||
|
||||
secret := box.Gate.SecretKey
|
||||
secret := box.Gate.AccessKey
|
||||
service, region := submatches["service"], submatches["region"]
|
||||
|
||||
signature := signStr(secret, service, region, signatureDateTime, policy)
|
||||
reqSignature := MultipartFormValue(r, "x-amz-signature")
|
||||
if signature != reqSignature {
|
||||
return nil, fmt.Errorf("%w: %s != %s", apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch),
|
||||
reqSignature, signature)
|
||||
if signature != MultipartFormValue(r, "x-amz-signature") {
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
|
||||
return &middleware.Box{AccessBox: box, Attributes: attrs}, nil
|
||||
return &Box{AccessBox: box}, nil
|
||||
}
|
||||
|
||||
func cloneRequest(r *http.Request, authHeader *AuthHeader) *http.Request {
|
||||
func cloneRequest(r *http.Request, authHeader *authHeader) *http.Request {
|
||||
otherRequest := r.Clone(context.TODO())
|
||||
otherRequest.Header = make(http.Header)
|
||||
|
||||
|
@ -314,27 +288,25 @@ func cloneRequest(r *http.Request, authHeader *AuthHeader) *http.Request {
|
|||
return otherRequest
|
||||
}
|
||||
|
||||
func (c *Center) checkSign(authHeader *AuthHeader, box *accessbox.Box, request *http.Request, signatureDateTime time.Time) error {
|
||||
awsCreds := credentials.NewStaticCredentials(authHeader.AccessKeyID, box.Gate.SecretKey, "")
|
||||
func (c *center) checkSign(authHeader *authHeader, box *accessbox.Box, request *http.Request, signatureDateTime time.Time) error {
|
||||
awsCreds := credentials.NewStaticCredentials(authHeader.AccessKeyID, box.Gate.AccessKey, "")
|
||||
signer := v4.NewSigner(awsCreds)
|
||||
signer.DisableURIPathEscaping = true
|
||||
|
||||
var signature string
|
||||
if authHeader.IsPresigned {
|
||||
now := time.Now()
|
||||
if signatureDateTime.Add(authHeader.Expiration).Before(now) {
|
||||
return fmt.Errorf("%w: expired: now %s, signature %s", apiErrors.GetAPIError(apiErrors.ErrExpiredPresignRequest),
|
||||
now.Format(time.RFC3339), signatureDateTime.Format(time.RFC3339))
|
||||
return apiErrors.GetAPIError(apiErrors.ErrExpiredPresignRequest)
|
||||
}
|
||||
if now.Before(signatureDateTime) {
|
||||
return fmt.Errorf("%w: signature time from the future: now %s, signature %s", apiErrors.GetAPIError(apiErrors.ErrBadRequest),
|
||||
now.Format(time.RFC3339), signatureDateTime.Format(time.RFC3339))
|
||||
return apiErrors.GetAPIError(apiErrors.ErrBadRequest)
|
||||
}
|
||||
if _, err := signer.Presign(request, nil, authHeader.Service, authHeader.Region, authHeader.Expiration, signatureDateTime); err != nil {
|
||||
return fmt.Errorf("failed to pre-sign temporary HTTP request: %w", err)
|
||||
}
|
||||
signature = request.URL.Query().Get(AmzSignature)
|
||||
} else {
|
||||
signer.DisableURIPathEscaping = true
|
||||
if _, err := signer.Sign(request, nil, authHeader.Service, authHeader.Region, signatureDateTime); err != nil {
|
||||
return fmt.Errorf("failed to sign temporary HTTP request: %w", err)
|
||||
}
|
||||
|
@ -342,8 +314,7 @@ func (c *Center) checkSign(authHeader *AuthHeader, box *accessbox.Box, request *
|
|||
}
|
||||
|
||||
if authHeader.SignatureV4 != signature {
|
||||
return fmt.Errorf("%w: %s != %s: headers %v", apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch),
|
||||
authHeader.SignatureV4, signature, authHeader.SignedFields)
|
||||
return apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
@ -5,26 +5,26 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAuthHeaderParse(t *testing.T) {
|
||||
defaultHeader := "AWS4-HMAC-SHA256 Credential=oid0cid/20210809/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2811ccb9e242f41426738fb1f"
|
||||
|
||||
center := &Center{
|
||||
center := ¢er{
|
||||
reg: NewRegexpMatcher(authorizationFieldRegexp),
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
header string
|
||||
err error
|
||||
expected *AuthHeader
|
||||
expected *authHeader
|
||||
}{
|
||||
{
|
||||
header: defaultHeader,
|
||||
err: nil,
|
||||
expected: &AuthHeader{
|
||||
expected: &authHeader{
|
||||
AccessKeyID: "oid0cid",
|
||||
Service: "s3",
|
||||
Region: "us-east-1",
|
||||
|
@ -45,7 +45,7 @@ func TestAuthHeaderParse(t *testing.T) {
|
|||
},
|
||||
} {
|
||||
authHeader, err := center.parseAuthHeader(tc.header)
|
||||
require.ErrorIs(t, err, tc.err, tc.header)
|
||||
require.Equal(t, tc.err, err, tc.header)
|
||||
require.Equal(t, tc.expected, authHeader, tc.header)
|
||||
}
|
||||
}
|
||||
|
@ -54,36 +54,36 @@ func TestAuthHeaderGetAddress(t *testing.T) {
|
|||
defaulErr := errors.GetAPIError(errors.ErrInvalidAccessKeyID)
|
||||
|
||||
for _, tc := range []struct {
|
||||
authHeader *AuthHeader
|
||||
authHeader *authHeader
|
||||
err error
|
||||
}{
|
||||
{
|
||||
authHeader: &AuthHeader{
|
||||
authHeader: &authHeader{
|
||||
AccessKeyID: "vWqF8cMDRbJcvnPLALoQGnABPPhw8NyYMcGsfDPfZJM0HrgjonN8CgFvCZ3kh9BUXw4W2tJ5E7EAGhueSF122HB",
|
||||
},
|
||||
err: nil,
|
||||
},
|
||||
{
|
||||
authHeader: &AuthHeader{
|
||||
authHeader: &authHeader{
|
||||
AccessKeyID: "vWqF8cMDRbJcvnPLALoQGnABPPhw8NyYMcGsfDPfZJMHrgjonN8CgFvCZ3kh9BUXw4W2tJ5E7EAGhueSF122HB",
|
||||
},
|
||||
err: defaulErr,
|
||||
},
|
||||
{
|
||||
authHeader: &AuthHeader{
|
||||
authHeader: &authHeader{
|
||||
AccessKeyID: "oid0cid",
|
||||
},
|
||||
err: defaulErr,
|
||||
},
|
||||
{
|
||||
authHeader: &AuthHeader{
|
||||
authHeader: &authHeader{
|
||||
AccessKeyID: "oidcid",
|
||||
},
|
||||
err: defaulErr,
|
||||
},
|
||||
} {
|
||||
_, err := getAddress(tc.authHeader.AccessKeyID)
|
||||
require.ErrorIs(t, err, tc.err, tc.authHeader.AccessKeyID)
|
||||
_, err := tc.authHeader.getAddress()
|
||||
require.Equal(t, tc.err, err, tc.authHeader.AccessKeyID)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -99,49 +99,3 @@ func TestSignature(t *testing.T) {
|
|||
signature := signStr(secret, "s3", "us-east-1", signTime, strToSign)
|
||||
require.Equal(t, "dfbe886241d9e369cf4b329ca0f15eb27306c97aa1022cc0bb5a914c4ef87634", signature)
|
||||
}
|
||||
|
||||
func TestCheckFormatContentSHA256(t *testing.T) {
|
||||
defaultErr := errors.GetAPIError(errors.ErrContentSHA256Mismatch)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
hash string
|
||||
error error
|
||||
}{
|
||||
{
|
||||
name: "invalid hash format: length and character",
|
||||
hash: "invalid-hash",
|
||||
error: defaultErr,
|
||||
},
|
||||
{
|
||||
name: "invalid hash format: length (63 characters)",
|
||||
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f7",
|
||||
error: defaultErr,
|
||||
},
|
||||
{
|
||||
name: "invalid hash format: character",
|
||||
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f7s",
|
||||
error: defaultErr,
|
||||
},
|
||||
{
|
||||
name: "unsigned payload",
|
||||
hash: "UNSIGNED-PAYLOAD",
|
||||
error: nil,
|
||||
},
|
||||
{
|
||||
name: "no hash",
|
||||
hash: "",
|
||||
error: nil,
|
||||
},
|
||||
{
|
||||
name: "correct hash format",
|
||||
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
|
||||
error: nil,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := checkFormatHashContentSHA256(tc.hash)
|
||||
require.ErrorIs(t, err, tc.error)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,47 +0,0 @@
|
|||
package auth
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/private/protocol/rest"
|
||||
)
|
||||
|
||||
type RequestData struct {
|
||||
Method string
|
||||
Endpoint string
|
||||
Bucket string
|
||||
Object string
|
||||
}
|
||||
|
||||
type PresignData struct {
|
||||
Service string
|
||||
Region string
|
||||
Lifetime time.Duration
|
||||
SignTime time.Time
|
||||
}
|
||||
|
||||
// PresignRequest forms pre-signed request to access objects without aws credentials.
|
||||
func PresignRequest(creds *credentials.Credentials, reqData RequestData, presignData PresignData) (*http.Request, error) {
|
||||
urlStr := fmt.Sprintf("%s/%s/%s", reqData.Endpoint, rest.EscapePath(reqData.Bucket, false), rest.EscapePath(reqData.Object, false))
|
||||
req, err := http.NewRequest(strings.ToUpper(reqData.Method), urlStr, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create new request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set(AmzDate, presignData.SignTime.Format("20060102T150405Z"))
|
||||
req.Header.Set(ContentTypeHdr, "text/plain")
|
||||
|
||||
signer := v4.NewSigner(creds)
|
||||
signer.DisableURIPathEscaping = true
|
||||
|
||||
if _, err = signer.Presign(req, nil, presignData.Service, presignData.Region, presignData.Lifetime, presignData.SignTime); err != nil {
|
||||
return nil, fmt.Errorf("presign: %w", err)
|
||||
}
|
||||
|
||||
return req, nil
|
||||
}
|
|
@ -1,94 +0,0 @@
|
|||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var _ tokens.Credentials = (*credentialsMock)(nil)
|
||||
|
||||
type credentialsMock struct {
|
||||
boxes map[string]*accessbox.Box
|
||||
}
|
||||
|
||||
func newTokensFrostfsMock() *credentialsMock {
|
||||
return &credentialsMock{
|
||||
boxes: make(map[string]*accessbox.Box),
|
||||
}
|
||||
}
|
||||
|
||||
func (m credentialsMock) addBox(addr oid.Address, box *accessbox.Box) {
|
||||
m.boxes[addr.String()] = box
|
||||
}
|
||||
|
||||
func (m credentialsMock) GetBox(_ context.Context, addr oid.Address) (*accessbox.Box, []object.Attribute, error) {
|
||||
box, ok := m.boxes[addr.String()]
|
||||
if !ok {
|
||||
return nil, nil, &apistatus.ObjectNotFound{}
|
||||
}
|
||||
|
||||
return box, nil, nil
|
||||
}
|
||||
|
||||
func (m credentialsMock) Put(context.Context, cid.ID, tokens.CredentialsParam) (oid.Address, error) {
|
||||
return oid.Address{}, nil
|
||||
}
|
||||
|
||||
func (m credentialsMock) Update(context.Context, oid.Address, tokens.CredentialsParam) (oid.Address, error) {
|
||||
return oid.Address{}, nil
|
||||
}
|
||||
|
||||
func TestCheckSign(t *testing.T) {
|
||||
var accessKeyAddr oid.Address
|
||||
err := accessKeyAddr.DecodeString("8N7CYBY74kxZXoyvA5UNdmovaXqFpwNfvEPsqaN81es2/3tDwq5tR8fByrJcyJwyiuYX7Dae8tyDT7pd8oaL1MBto")
|
||||
require.NoError(t, err)
|
||||
|
||||
accessKeyID := strings.ReplaceAll(accessKeyAddr.String(), "/", "0")
|
||||
secretKey := "713d0a0b9efc7d22923e17b0402a6a89b4273bc711c8bacb2da1b643d0006aeb"
|
||||
awsCreds := credentials.NewStaticCredentials(accessKeyID, secretKey, "")
|
||||
|
||||
reqData := RequestData{
|
||||
Method: "GET",
|
||||
Endpoint: "http://localhost:8084",
|
||||
Bucket: "my-bucket",
|
||||
Object: "@obj/name",
|
||||
}
|
||||
presignData := PresignData{
|
||||
Service: "s3",
|
||||
Region: "spb",
|
||||
Lifetime: 10 * time.Minute,
|
||||
SignTime: time.Now().UTC(),
|
||||
}
|
||||
|
||||
req, err := PresignRequest(awsCreds, reqData, presignData)
|
||||
require.NoError(t, err)
|
||||
|
||||
expBox := &accessbox.Box{
|
||||
Gate: &accessbox.GateData{
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
}
|
||||
|
||||
mock := newTokensFrostfsMock()
|
||||
mock.addBox(accessKeyAddr, expBox)
|
||||
|
||||
c := &Center{
|
||||
cli: mock,
|
||||
reg: NewRegexpMatcher(authorizationFieldRegexp),
|
||||
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
||||
}
|
||||
box, err := c.Authenticate(req)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, expBox, box.AccessBox)
|
||||
}
|
|
@ -790,8 +790,6 @@ const doubleSpace = " "
|
|||
|
||||
// stripExcessSpaces will rewrite the passed in slice's string values to not
|
||||
// contain multiple side-by-side spaces.
|
||||
//
|
||||
//nolint:revive
|
||||
func stripExcessSpaces(vals []string) {
|
||||
var j, k, l, m, spaces int
|
||||
for i, str := range vals {
|
||||
|
|
5
api/cache/access_control.go
vendored
5
api/cache/access_control.go
vendored
|
@ -4,8 +4,7 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -47,7 +46,7 @@ func (o *AccessControlCache) Get(owner user.ID, key string) bool {
|
|||
|
||||
result, ok := entry.(bool)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return false
|
||||
}
|
||||
|
|
38
api/cache/accessbox.go
vendored
38
api/cache/accessbox.go
vendored
|
@ -4,10 +4,8 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -25,12 +23,6 @@ type (
|
|||
Lifetime time.Duration
|
||||
Logger *zap.Logger
|
||||
}
|
||||
|
||||
AccessBoxCacheValue struct {
|
||||
Box *accessbox.Box
|
||||
Attributes []object.Attribute
|
||||
PutTime time.Time
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -49,23 +41,23 @@ func DefaultAccessBoxConfig(logger *zap.Logger) *Config {
|
|||
}
|
||||
}
|
||||
|
||||
// NewAccessBoxCache creates an object of AccessBoxCache.
|
||||
// NewAccessBoxCache creates an object of BucketCache.
|
||||
func NewAccessBoxCache(config *Config) *AccessBoxCache {
|
||||
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||
|
||||
return &AccessBoxCache{cache: gc, logger: config.Logger}
|
||||
}
|
||||
|
||||
// Get returns a cached accessbox.
|
||||
func (o *AccessBoxCache) Get(address oid.Address) *AccessBoxCacheValue {
|
||||
// Get returns a cached object.
|
||||
func (o *AccessBoxCache) Get(address oid.Address) *accessbox.Box {
|
||||
entry, err := o.cache.Get(address)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := entry.(*AccessBoxCacheValue)
|
||||
result, ok := entry.(*accessbox.Box)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
@ -73,17 +65,7 @@ func (o *AccessBoxCache) Get(address oid.Address) *AccessBoxCacheValue {
|
|||
return result
|
||||
}
|
||||
|
||||
// Put stores an accessbox to cache.
|
||||
func (o *AccessBoxCache) Put(address oid.Address, box *accessbox.Box, attrs []object.Attribute) error {
|
||||
val := &AccessBoxCacheValue{
|
||||
Box: box,
|
||||
Attributes: attrs,
|
||||
PutTime: time.Now(),
|
||||
}
|
||||
return o.cache.Set(address, val)
|
||||
}
|
||||
|
||||
// Delete removes an accessbox from cache.
|
||||
func (o *AccessBoxCache) Delete(address oid.Address) {
|
||||
o.cache.Remove(address)
|
||||
// Put stores an object to cache.
|
||||
func (o *AccessBoxCache) Put(address oid.Address, box *accessbox.Box) error {
|
||||
return o.cache.Set(address, box)
|
||||
}
|
||||
|
|
19
api/cache/buckets.go
vendored
19
api/cache/buckets.go
vendored
|
@ -4,8 +4,7 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -39,15 +38,15 @@ func NewBucketCache(config *Config) *BucketCache {
|
|||
}
|
||||
|
||||
// Get returns a cached object.
|
||||
func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo {
|
||||
entry, err := o.cache.Get(formKey(ns, bktName))
|
||||
func (o *BucketCache) Get(key string) *data.BucketInfo {
|
||||
entry, err := o.cache.Get(key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := entry.(*data.BucketInfo)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
@ -57,14 +56,10 @@ func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo {
|
|||
|
||||
// Put puts an object to cache.
|
||||
func (o *BucketCache) Put(bkt *data.BucketInfo) error {
|
||||
return o.cache.Set(formKey(bkt.Zone, bkt.Name), bkt)
|
||||
return o.cache.Set(bkt.Name, bkt)
|
||||
}
|
||||
|
||||
// Delete deletes an object from cache.
|
||||
func (o *BucketCache) Delete(bkt *data.BucketInfo) bool {
|
||||
return o.cache.Remove(formKey(bkt.Zone, bkt.Name))
|
||||
}
|
||||
|
||||
func formKey(zone, name string) string {
|
||||
return name + "." + zone
|
||||
func (o *BucketCache) Delete(key string) bool {
|
||||
return o.cache.Remove(key)
|
||||
}
|
||||
|
|
62
api/cache/cache_test.go
vendored
62
api/cache/cache_test.go
vendored
|
@ -3,14 +3,10 @@ package cache
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neo-go/pkg/util"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
cidtest "github.com/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
oidtest "github.com/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zaptest/observer"
|
||||
|
@ -22,13 +18,11 @@ func TestAccessBoxCacheType(t *testing.T) {
|
|||
|
||||
addr := oidtest.Address()
|
||||
box := &accessbox.Box{}
|
||||
var attrs []object.Attribute
|
||||
|
||||
err := cache.Put(addr, box, attrs)
|
||||
err := cache.Put(addr, box)
|
||||
require.NoError(t, err)
|
||||
val := cache.Get(addr)
|
||||
require.Equal(t, box, val.Box)
|
||||
require.Equal(t, attrs, val.Attributes)
|
||||
require.Equal(t, box, val)
|
||||
require.Equal(t, 0, observedLog.Len())
|
||||
|
||||
err = cache.cache.Set(addr, "tmp")
|
||||
|
@ -44,13 +38,13 @@ func TestBucketsCacheType(t *testing.T) {
|
|||
|
||||
err := cache.Put(bktInfo)
|
||||
require.NoError(t, err)
|
||||
val := cache.Get("", bktInfo.Name)
|
||||
val := cache.Get(bktInfo.Name)
|
||||
require.Equal(t, bktInfo, val)
|
||||
require.Equal(t, 0, observedLog.Len())
|
||||
|
||||
err = cache.cache.Set(bktInfo.Name+"."+bktInfo.Zone, "tmp")
|
||||
err = cache.cache.Set(bktInfo.Name, "tmp")
|
||||
require.NoError(t, err)
|
||||
assertInvalidCacheEntry(t, cache.Get(bktInfo.Zone, bktInfo.Name), observedLog)
|
||||
assertInvalidCacheEntry(t, cache.Get(bktInfo.Name), observedLog)
|
||||
}
|
||||
|
||||
func TestObjectNamesCacheType(t *testing.T) {
|
||||
|
@ -200,44 +194,6 @@ func TestNotificationConfigurationCacheType(t *testing.T) {
|
|||
assertInvalidCacheEntry(t, cache.GetNotificationConfiguration(key), observedLog)
|
||||
}
|
||||
|
||||
func TestFrostFSIDSubjectCacheType(t *testing.T) {
|
||||
logger, observedLog := getObservedLogger()
|
||||
cache := NewFrostfsIDCache(DefaultFrostfsIDConfig(logger))
|
||||
|
||||
key, err := util.Uint160DecodeStringLE("4ea976429703418ef00fc4912a409b6a0b973034")
|
||||
require.NoError(t, err)
|
||||
value := &client.SubjectExtended{}
|
||||
|
||||
err = cache.PutSubject(key, value)
|
||||
require.NoError(t, err)
|
||||
val := cache.GetSubject(key)
|
||||
require.Equal(t, value, val)
|
||||
require.Equal(t, 0, observedLog.Len())
|
||||
|
||||
err = cache.cache.Set(key, "tmp")
|
||||
require.NoError(t, err)
|
||||
assertInvalidCacheEntry(t, cache.GetSubject(key), observedLog)
|
||||
}
|
||||
|
||||
func TestFrostFSIDUserKeyCacheType(t *testing.T) {
|
||||
logger, observedLog := getObservedLogger()
|
||||
cache := NewFrostfsIDCache(DefaultFrostfsIDConfig(logger))
|
||||
|
||||
ns, name := "ns", "name"
|
||||
value, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
err = cache.PutUserKey(ns, name, value.PublicKey())
|
||||
require.NoError(t, err)
|
||||
val := cache.GetUserKey(ns, name)
|
||||
require.Equal(t, value.PublicKey(), val)
|
||||
require.Equal(t, 0, observedLog.Len())
|
||||
|
||||
err = cache.cache.Set(ns+"/"+name, "tmp")
|
||||
require.NoError(t, err)
|
||||
assertInvalidCacheEntry(t, cache.GetUserKey(ns, name), observedLog)
|
||||
}
|
||||
|
||||
func assertInvalidCacheEntry(t *testing.T, val interface{}, observedLog *observer.ObservedLogs) {
|
||||
require.Nil(t, val)
|
||||
require.Equal(t, 1, observedLog.Len())
|
||||
|
|
77
api/cache/frostfsid.go
vendored
77
api/cache/frostfsid.go
vendored
|
@ -1,77 +0,0 @@
|
|||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neo-go/pkg/util"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// FrostfsIDCache provides lru cache for frostfsid contract.
|
||||
type FrostfsIDCache struct {
|
||||
cache gcache.Cache
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
const (
|
||||
// DefaultFrostfsIDCacheSize is a default maximum number of entries in cache.
|
||||
DefaultFrostfsIDCacheSize = 1e4
|
||||
// DefaultFrostfsIDCacheLifetime is a default lifetime of entries in cache.
|
||||
DefaultFrostfsIDCacheLifetime = time.Minute
|
||||
)
|
||||
|
||||
// DefaultFrostfsIDConfig returns new default cache expiration values.
|
||||
func DefaultFrostfsIDConfig(logger *zap.Logger) *Config {
|
||||
return &Config{
|
||||
Size: DefaultFrostfsIDCacheSize,
|
||||
Lifetime: DefaultFrostfsIDCacheLifetime,
|
||||
Logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// NewFrostfsIDCache creates an object of FrostfsIDCache.
|
||||
func NewFrostfsIDCache(config *Config) *FrostfsIDCache {
|
||||
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||
return &FrostfsIDCache{cache: gc, logger: config.Logger}
|
||||
}
|
||||
|
||||
// GetSubject returns a cached client.SubjectExtended. Returns nil if value is missing.
|
||||
func (c *FrostfsIDCache) GetSubject(key util.Uint160) *client.SubjectExtended {
|
||||
return get[client.SubjectExtended](c, key)
|
||||
}
|
||||
|
||||
// PutSubject puts a client.SubjectExtended to cache.
|
||||
func (c *FrostfsIDCache) PutSubject(key util.Uint160, subject *client.SubjectExtended) error {
|
||||
return c.cache.Set(key, subject)
|
||||
}
|
||||
|
||||
// GetUserKey returns a cached *keys.PublicKey. Returns nil if value is missing.
|
||||
func (c *FrostfsIDCache) GetUserKey(ns, name string) *keys.PublicKey {
|
||||
return get[keys.PublicKey](c, ns+"/"+name)
|
||||
}
|
||||
|
||||
// PutUserKey puts a client.SubjectExtended to cache.
|
||||
func (c *FrostfsIDCache) PutUserKey(ns, name string, userKey *keys.PublicKey) error {
|
||||
return c.cache.Set(ns+"/"+name, userKey)
|
||||
}
|
||||
|
||||
func get[T any](c *FrostfsIDCache, key any) *T {
|
||||
entry, err := c.cache.Get(key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := entry.(*T)
|
||||
if !ok {
|
||||
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
107
api/cache/listsession.go
vendored
107
api/cache/listsession.go
vendored
|
@ -1,107 +0,0 @@
|
|||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type (
|
||||
// ListSessionCache contains cache for list session (during pagination).
|
||||
ListSessionCache struct {
|
||||
cache gcache.Cache
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// ListSessionKey is a key to find a ListSessionCache's entry.
|
||||
ListSessionKey struct {
|
||||
cid cid.ID
|
||||
prefix string
|
||||
token string
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultListSessionCacheLifetime is a default lifetime of entries in cache of ListObjects.
|
||||
DefaultListSessionCacheLifetime = time.Second * 60
|
||||
// DefaultListSessionCacheSize is a default size of cache of ListObjects.
|
||||
DefaultListSessionCacheSize = 100
|
||||
)
|
||||
|
||||
// DefaultListSessionConfig returns new default cache expiration values.
|
||||
func DefaultListSessionConfig(logger *zap.Logger) *Config {
|
||||
return &Config{
|
||||
Size: DefaultListSessionCacheSize,
|
||||
Lifetime: DefaultListSessionCacheLifetime,
|
||||
Logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (k *ListSessionKey) String() string {
|
||||
return k.cid.EncodeToString() + k.prefix + k.token
|
||||
}
|
||||
|
||||
// NewListSessionCache is a constructor which creates an object of ListObjectsCache with the given lifetime of entries.
|
||||
func NewListSessionCache(config *Config) *ListSessionCache {
|
||||
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).EvictedFunc(func(_ interface{}, val interface{}) {
|
||||
session, ok := val.(*data.ListSession)
|
||||
if !ok {
|
||||
config.Logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", val)),
|
||||
zap.String("expected", fmt.Sprintf("%T", session)))
|
||||
}
|
||||
|
||||
if !session.Acquired.Load() {
|
||||
session.Cancel()
|
||||
}
|
||||
}).Build()
|
||||
return &ListSessionCache{cache: gc, logger: config.Logger}
|
||||
}
|
||||
|
||||
// GetListSession returns a list of ObjectInfo.
|
||||
func (l *ListSessionCache) GetListSession(key ListSessionKey) *data.ListSession {
|
||||
entry, err := l.cache.Get(key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := entry.(*data.ListSession)
|
||||
if !ok {
|
||||
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// PutListSession puts a list of object versions to cache.
|
||||
func (l *ListSessionCache) PutListSession(key ListSessionKey, session *data.ListSession) error {
|
||||
s := l.GetListSession(key)
|
||||
if s != nil && s != session {
|
||||
if !s.Acquired.Load() {
|
||||
s.Cancel()
|
||||
}
|
||||
}
|
||||
return l.cache.Set(key, session)
|
||||
}
|
||||
|
||||
// DeleteListSession removes key from cache.
|
||||
func (l *ListSessionCache) DeleteListSession(key ListSessionKey) {
|
||||
l.cache.Remove(key)
|
||||
}
|
||||
|
||||
// CreateListSessionCacheKey returns ListSessionKey with the given CID, prefix and token.
|
||||
func CreateListSessionCacheKey(cnr cid.ID, prefix, token string) ListSessionKey {
|
||||
p := ListSessionKey{
|
||||
cid: cnr,
|
||||
prefix: prefix,
|
||||
token: token,
|
||||
}
|
||||
|
||||
return p
|
||||
}
|
5
api/cache/names.go
vendored
5
api/cache/names.go
vendored
|
@ -4,8 +4,7 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -49,7 +48,7 @@ func (o *ObjectsNameCache) Get(key string) *oid.Address {
|
|||
|
||||
result, ok := entry.(oid.Address)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
|
7
api/cache/objects.go
vendored
7
api/cache/objects.go
vendored
|
@ -4,9 +4,8 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -48,7 +47,7 @@ func (o *ObjectsCache) GetObject(address oid.Address) *data.ExtendedObjectInfo {
|
|||
|
||||
result, ok := entry.(*data.ExtendedObjectInfo)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
|
6
api/cache/objects_test.go
vendored
6
api/cache/objects_test.go
vendored
|
@ -4,9 +4,9 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
objecttest "github.com/TrueCloudLab/frostfs-sdk-go/object/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
|
9
api/cache/objectslist.go
vendored
9
api/cache/objectslist.go
vendored
|
@ -6,9 +6,8 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -76,7 +75,7 @@ func (l *ObjectsListCache) GetVersions(key ObjectsListKey) []*data.NodeVersion {
|
|||
|
||||
result, ok := entry.([]*data.NodeVersion)
|
||||
if !ok {
|
||||
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
l.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
@ -95,7 +94,7 @@ func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string,
|
|||
for _, key := range keys {
|
||||
k, ok := key.(ObjectsListKey)
|
||||
if !ok {
|
||||
l.logger.Warn(logs.InvalidCacheKeyType, zap.String("actual", fmt.Sprintf("%T", key)),
|
||||
l.logger.Warn("invalid cache key type", zap.String("actual", fmt.Sprintf("%T", key)),
|
||||
zap.String("expected", fmt.Sprintf("%T", k)))
|
||||
continue
|
||||
}
|
||||
|
|
6
api/cache/objectslist_test.go
vendored
6
api/cache/objectslist_test.go
vendored
|
@ -4,9 +4,9 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
cidtest "github.com/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
oidtest "github.com/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
|
72
api/cache/policy.go
vendored
72
api/cache/policy.go
vendored
|
@ -1,72 +0,0 @@
|
|||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// MorphPolicyCache provides lru cache for listing policies stored in policy contract.
|
||||
type MorphPolicyCache struct {
|
||||
cache gcache.Cache
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
type MorphPolicyCacheKey struct {
|
||||
Target engine.Target
|
||||
Name chain.Name
|
||||
}
|
||||
|
||||
const (
|
||||
// DefaultMorphPolicyCacheSize is a default maximum number of entries in cache.
|
||||
DefaultMorphPolicyCacheSize = 1e4
|
||||
// DefaultMorphPolicyCacheLifetime is a default lifetime of entries in cache.
|
||||
DefaultMorphPolicyCacheLifetime = time.Minute
|
||||
)
|
||||
|
||||
// DefaultMorphPolicyConfig returns new default cache expiration values.
|
||||
func DefaultMorphPolicyConfig(logger *zap.Logger) *Config {
|
||||
return &Config{
|
||||
Size: DefaultMorphPolicyCacheSize,
|
||||
Lifetime: DefaultMorphPolicyCacheLifetime,
|
||||
Logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// NewMorphPolicyCache creates an object of MorphPolicyCache.
|
||||
func NewMorphPolicyCache(config *Config) *MorphPolicyCache {
|
||||
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||
return &MorphPolicyCache{cache: gc, logger: config.Logger}
|
||||
}
|
||||
|
||||
// Get returns a cached object. Returns nil if value is missing.
|
||||
func (o *MorphPolicyCache) Get(key MorphPolicyCacheKey) []*chain.Chain {
|
||||
entry, err := o.cache.Get(key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := entry.([]*chain.Chain)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// Put puts an object to cache.
|
||||
func (o *MorphPolicyCache) Put(key MorphPolicyCacheKey, list []*chain.Chain) error {
|
||||
return o.cache.Set(key, list)
|
||||
}
|
||||
|
||||
// Delete deletes an object from cache.
|
||||
func (o *MorphPolicyCache) Delete(key MorphPolicyCacheKey) bool {
|
||||
return o.cache.Remove(key)
|
||||
}
|
11
api/cache/system.go
vendored
11
api/cache/system.go
vendored
|
@ -4,8 +4,7 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -49,7 +48,7 @@ func (o *SystemCache) GetObject(key string) *data.ObjectInfo {
|
|||
|
||||
result, ok := entry.(*data.ObjectInfo)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
@ -80,7 +79,7 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
|
|||
|
||||
result, ok := entry.(*data.CORSConfiguration)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
@ -96,7 +95,7 @@ func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
|||
|
||||
result, ok := entry.(*data.BucketSettings)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
@ -112,7 +111,7 @@ func (o *SystemCache) GetNotificationConfiguration(key string) *data.Notificatio
|
|||
|
||||
result, ok := entry.(*data.NotificationConfiguration)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -2,13 +2,11 @@ package data
|
|||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -24,48 +22,44 @@ const (
|
|||
type (
|
||||
// BucketInfo stores basic bucket data.
|
||||
BucketInfo struct {
|
||||
Name string // container name from system attribute
|
||||
Zone string // container zone from system attribute
|
||||
CID cid.ID
|
||||
Owner user.ID
|
||||
Created time.Time
|
||||
LocationConstraint string
|
||||
ObjectLockEnabled bool
|
||||
HomomorphicHashDisabled bool
|
||||
APEEnabled bool
|
||||
Name string // container name from system attribute
|
||||
Zone string // container zone from system attribute
|
||||
CID cid.ID
|
||||
Owner user.ID
|
||||
Created time.Time
|
||||
LocationConstraint string
|
||||
ObjectLockEnabled bool
|
||||
}
|
||||
|
||||
// ObjectInfo holds S3 object data.
|
||||
ObjectInfo struct {
|
||||
ID oid.ID
|
||||
CID cid.ID
|
||||
ID oid.ID
|
||||
CID cid.ID
|
||||
IsDir bool
|
||||
IsDeleteMarker bool
|
||||
|
||||
Bucket string
|
||||
Name string
|
||||
Size uint64
|
||||
ContentType string
|
||||
Created time.Time
|
||||
CreationEpoch uint64
|
||||
HashSum string
|
||||
MD5Sum string
|
||||
Owner user.ID
|
||||
Headers map[string]string
|
||||
Bucket string
|
||||
Name string
|
||||
Size int64
|
||||
ContentType string
|
||||
Created time.Time
|
||||
HashSum string
|
||||
Owner user.ID
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
// NotificationInfo store info to send s3 notification.
|
||||
NotificationInfo struct {
|
||||
Name string
|
||||
Version string
|
||||
Size uint64
|
||||
Size int64
|
||||
HashSum string
|
||||
}
|
||||
|
||||
// BucketSettings stores settings such as versioning.
|
||||
BucketSettings struct {
|
||||
Versioning string
|
||||
LockConfiguration *ObjectLockConfiguration
|
||||
CannedACL string
|
||||
OwnerKey *keys.PublicKey
|
||||
Versioning string `json:"versioning"`
|
||||
LockConfiguration *ObjectLockConfiguration `json:"lock_configuration"`
|
||||
}
|
||||
|
||||
// CORSConfiguration stores CORS configuration of a request.
|
||||
|
@ -83,23 +77,15 @@ type (
|
|||
ExposeHeaders []string `xml:"ExposeHeader" json:"ExposeHeaders"`
|
||||
MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty" json:"MaxAgeSeconds,omitempty"`
|
||||
}
|
||||
|
||||
// ObjectVersion stores object version info.
|
||||
ObjectVersion struct {
|
||||
BktInfo *BucketInfo
|
||||
ObjectName string
|
||||
VersionID string
|
||||
NoErrorOnDeleteMarker bool
|
||||
}
|
||||
)
|
||||
|
||||
// NotificationInfoFromObject creates new NotificationInfo from ObjectInfo.
|
||||
func NotificationInfoFromObject(objInfo *ObjectInfo, md5Enabled bool) *NotificationInfo {
|
||||
func NotificationInfoFromObject(objInfo *ObjectInfo) *NotificationInfo {
|
||||
return &NotificationInfo{
|
||||
Name: objInfo.Name,
|
||||
Version: objInfo.VersionID(),
|
||||
Size: objInfo.Size,
|
||||
HashSum: Quote(objInfo.ETag(md5Enabled)),
|
||||
HashSum: objInfo.HashSum,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -128,13 +114,6 @@ func (o *ObjectInfo) Address() oid.Address {
|
|||
return addr
|
||||
}
|
||||
|
||||
func (o *ObjectInfo) ETag(md5Enabled bool) string {
|
||||
if md5Enabled && len(o.MD5Sum) > 0 {
|
||||
return o.MD5Sum
|
||||
}
|
||||
return o.HashSum
|
||||
}
|
||||
|
||||
func (b BucketSettings) Unversioned() bool {
|
||||
return b.Versioning == VersioningUnversioned
|
||||
}
|
||||
|
@ -146,11 +125,3 @@ func (b BucketSettings) VersioningEnabled() bool {
|
|||
func (b BucketSettings) VersioningSuspended() bool {
|
||||
return b.Versioning == VersioningSuspended
|
||||
}
|
||||
|
||||
func Quote(val string) string {
|
||||
return "\"" + val + "\""
|
||||
}
|
||||
|
||||
func UnQuote(val string) string {
|
||||
return strings.Trim(val, "\"")
|
||||
}
|
||||
|
|
|
@ -1,19 +0,0 @@
|
|||
package data
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
type VersionsStream interface {
|
||||
Next(ctx context.Context) (*NodeVersion, error)
|
||||
}
|
||||
|
||||
type ListSession struct {
|
||||
Next []*ExtendedNodeVersion
|
||||
Stream VersionsStream
|
||||
NamesMap map[string]struct{}
|
||||
Context context.Context
|
||||
Cancel context.CancelFunc
|
||||
Acquired atomic.Bool
|
||||
}
|
|
@ -1,10 +1,7 @@
|
|||
package data
|
||||
|
||||
import "encoding/xml"
|
||||
|
||||
type (
|
||||
NotificationConfiguration struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NotificationConfiguration" json:"-"`
|
||||
QueueConfigurations []QueueConfiguration `xml:"QueueConfiguration" json:"QueueConfigurations"`
|
||||
// Not supported topics
|
||||
TopicConfigurations []TopicConfiguration `xml:"TopicConfiguration" json:"TopicConfigurations"`
|
||||
|
|
|
@ -1,30 +0,0 @@
|
|||
package data
|
||||
|
||||
import "encoding/xml"
|
||||
|
||||
// Tagging contains tag set.
|
||||
type Tagging struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"`
|
||||
TagSet []Tag `xml:"TagSet>Tag"`
|
||||
}
|
||||
|
||||
// Tag is an AWS key-value tag.
|
||||
type Tag struct {
|
||||
Key string
|
||||
Value string
|
||||
}
|
||||
|
||||
type GetObjectTaggingParams struct {
|
||||
ObjectVersion *ObjectVersion
|
||||
|
||||
// NodeVersion can be nil. If not nil we save one request to tree service.
|
||||
NodeVersion *NodeVersion // optional
|
||||
}
|
||||
|
||||
type PutObjectTaggingParams struct {
|
||||
ObjectVersion *ObjectVersion
|
||||
TagSet map[string]string
|
||||
|
||||
// NodeVersion can be nil. If not nil we save one request to tree service.
|
||||
NodeVersion *NodeVersion // optional
|
||||
}
|
110
api/data/tree.go
110
api/data/tree.go
|
@ -4,9 +4,9 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -16,31 +16,19 @@ const (
|
|||
// NodeVersion represent node from tree service.
|
||||
type NodeVersion struct {
|
||||
BaseNodeVersion
|
||||
DeleteMarker *DeleteMarkerInfo
|
||||
IsUnversioned bool
|
||||
IsCombined bool
|
||||
}
|
||||
|
||||
// ExtendedNodeVersion contains additional node info to be able to sort versions by timestamp.
|
||||
type ExtendedNodeVersion struct {
|
||||
NodeVersion *NodeVersion
|
||||
IsLatest bool
|
||||
DirName string
|
||||
func (v NodeVersion) IsDeleteMarker() bool {
|
||||
return v.DeleteMarker != nil
|
||||
}
|
||||
|
||||
func (e ExtendedNodeVersion) Version() string {
|
||||
if e.NodeVersion.IsUnversioned {
|
||||
return UnversionedObjectVersionID
|
||||
}
|
||||
|
||||
return e.NodeVersion.OID.EncodeToString()
|
||||
}
|
||||
|
||||
func (e ExtendedNodeVersion) Name() string {
|
||||
if e.DirName != "" {
|
||||
return e.DirName
|
||||
}
|
||||
|
||||
return e.NodeVersion.FilePath
|
||||
// DeleteMarkerInfo is used to save object info if node in the tree service is delete marker.
|
||||
// We need this information because the "delete marker" object is no longer stored in FrostFS.
|
||||
type DeleteMarkerInfo struct {
|
||||
Created time.Time
|
||||
Owner user.ID
|
||||
}
|
||||
|
||||
// ExtendedObjectInfo contains additional node info to be able to sort versions by timestamp.
|
||||
|
@ -61,35 +49,13 @@ func (e ExtendedObjectInfo) Version() string {
|
|||
// BaseNodeVersion is minimal node info from tree service.
|
||||
// Basically used for "system" object.
|
||||
type BaseNodeVersion struct {
|
||||
ID uint64
|
||||
ParenID uint64
|
||||
OID oid.ID
|
||||
Timestamp uint64
|
||||
Size uint64
|
||||
ETag string
|
||||
MD5 string
|
||||
FilePath string
|
||||
Created *time.Time
|
||||
Owner *user.ID
|
||||
IsDeleteMarker bool
|
||||
}
|
||||
|
||||
func (v *BaseNodeVersion) GetETag(md5Enabled bool) string {
|
||||
if md5Enabled && len(v.MD5) > 0 {
|
||||
return v.MD5
|
||||
}
|
||||
return v.ETag
|
||||
}
|
||||
|
||||
// IsFilledExtra returns true is node was created by version of gate v0.29.x and later.
|
||||
func (v BaseNodeVersion) IsFilledExtra() bool {
|
||||
return v.Created != nil && v.Owner != nil
|
||||
}
|
||||
|
||||
func (v *BaseNodeVersion) FillExtra(owner *user.ID, created *time.Time, realSize uint64) {
|
||||
v.Owner = owner
|
||||
v.Created = created
|
||||
v.Size = realSize
|
||||
ID uint64
|
||||
ParenID uint64
|
||||
OID oid.ID
|
||||
Timestamp uint64
|
||||
Size int64
|
||||
ETag string
|
||||
FilePath string
|
||||
}
|
||||
|
||||
type ObjectTaggingInfo struct {
|
||||
|
@ -102,39 +68,29 @@ type ObjectTaggingInfo struct {
|
|||
type MultipartInfo struct {
|
||||
// ID is node id in tree service.
|
||||
// It's ignored when creating a new multipart upload.
|
||||
ID uint64
|
||||
Key string
|
||||
UploadID string
|
||||
Owner user.ID
|
||||
Created time.Time
|
||||
Meta map[string]string
|
||||
CopiesNumbers []uint32
|
||||
Finished bool
|
||||
ID uint64
|
||||
Key string
|
||||
UploadID string
|
||||
Owner user.ID
|
||||
Created time.Time
|
||||
Meta map[string]string
|
||||
CopiesNumber uint32
|
||||
}
|
||||
|
||||
// PartInfo is upload information about part.
|
||||
type PartInfo struct {
|
||||
Key string `json:"key"`
|
||||
UploadID string `json:"uploadId"`
|
||||
Number int `json:"number"`
|
||||
OID oid.ID `json:"oid"`
|
||||
Size uint64 `json:"size"`
|
||||
ETag string `json:"etag"`
|
||||
MD5 string `json:"md5"`
|
||||
Created time.Time `json:"created"`
|
||||
Key string
|
||||
UploadID string
|
||||
Number int
|
||||
OID oid.ID
|
||||
Size int64
|
||||
ETag string
|
||||
Created time.Time
|
||||
}
|
||||
|
||||
// ToHeaderString form short part representation to use in S3-Completed-Parts header.
|
||||
func (p *PartInfo) ToHeaderString() string {
|
||||
// ETag value contains SHA256 checksum which is used while getting object parts attributes.
|
||||
return strconv.Itoa(p.Number) + "-" + strconv.FormatUint(p.Size, 10) + "-" + p.ETag
|
||||
}
|
||||
|
||||
func (p *PartInfo) GetETag(md5Enabled bool) string {
|
||||
if md5Enabled && len(p.MD5) > 0 {
|
||||
return p.MD5
|
||||
}
|
||||
return p.ETag
|
||||
return strconv.Itoa(p.Number) + "-" + strconv.FormatInt(p.Size, 10) + "-" + p.ETag
|
||||
}
|
||||
|
||||
// LockInfo is lock information to create appropriate tree node.
|
||||
|
|
|
@ -3,8 +3,6 @@ package errors
|
|||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||
)
|
||||
|
||||
type (
|
||||
|
@ -26,7 +24,6 @@ type (
|
|||
const (
|
||||
_ ErrorCode = iota
|
||||
ErrAccessDenied
|
||||
ErrAccessControlListNotSupported
|
||||
ErrBadDigest
|
||||
ErrEntityTooSmall
|
||||
ErrEntityTooLarge
|
||||
|
@ -74,7 +71,6 @@ const (
|
|||
ErrInvalidArgument
|
||||
ErrInvalidTagKey
|
||||
ErrInvalidTagValue
|
||||
ErrInvalidTagKeyUniqueness
|
||||
ErrInvalidTagsSizeExceed
|
||||
ErrNotImplemented
|
||||
ErrPreconditionFailed
|
||||
|
@ -91,7 +87,6 @@ const (
|
|||
ErrBucketNotEmpty
|
||||
ErrAllAccessDisabled
|
||||
ErrMalformedPolicy
|
||||
ErrMalformedPolicyNotPrincipal
|
||||
ErrMissingFields
|
||||
ErrMissingCredTag
|
||||
ErrCredMalformed
|
||||
|
@ -151,7 +146,6 @@ const (
|
|||
ErrInvalidEncryptionAlgorithm
|
||||
ErrInvalidSSECustomerKey
|
||||
ErrMissingSSECustomerKey
|
||||
ErrMissingSSECustomerAlgorithm
|
||||
ErrMissingSSECustomerKeyMD5
|
||||
ErrSSECustomerKeyMD5Mismatch
|
||||
ErrInvalidSSECustomerParameters
|
||||
|
@ -181,12 +175,9 @@ const (
|
|||
// Add new extended error codes here.
|
||||
ErrInvalidObjectName
|
||||
ErrOperationTimedOut
|
||||
ErrGatewayTimeout
|
||||
ErrOperationMaxedOut
|
||||
ErrInvalidRequest
|
||||
ErrInvalidRequestLargeCopy
|
||||
ErrInvalidStorageClass
|
||||
VersionIDMarkerWithoutKeyMarker
|
||||
|
||||
ErrMalformedJSON
|
||||
ErrInsecureClientRequest
|
||||
|
@ -318,12 +309,6 @@ var errorCodes = errorCodeMap{
|
|||
Description: "Invalid storage class.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
VersionIDMarkerWithoutKeyMarker: {
|
||||
ErrCode: VersionIDMarkerWithoutKeyMarker,
|
||||
Code: "VersionIDMarkerWithoutKeyMarker",
|
||||
Description: "A version-id marker cannot be specified without a key marker.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidRequestBody: {
|
||||
ErrCode: ErrInvalidRequestBody,
|
||||
Code: "InvalidArgument",
|
||||
|
@ -333,7 +318,7 @@ var errorCodes = errorCodeMap{
|
|||
ErrInvalidMaxUploads: {
|
||||
ErrCode: ErrInvalidMaxUploads,
|
||||
Code: "InvalidArgument",
|
||||
Description: "Argument max-uploads must be an integer from 1 to 1000",
|
||||
Description: "Argument max-uploads must be an integer between 0 and 2147483647",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidMaxKeys: {
|
||||
|
@ -378,12 +363,6 @@ var errorCodes = errorCodeMap{
|
|||
Description: "Access Denied.",
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
},
|
||||
ErrAccessControlListNotSupported: {
|
||||
ErrCode: ErrAccessControlListNotSupported,
|
||||
Code: "AccessControlListNotSupported",
|
||||
Description: "The bucket does not allow ACLs.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrBadDigest: {
|
||||
ErrCode: ErrBadDigest,
|
||||
Code: "BadDigest",
|
||||
|
@ -543,19 +522,13 @@ var errorCodes = errorCodeMap{
|
|||
ErrInvalidTagKey: {
|
||||
ErrCode: ErrInvalidTagKey,
|
||||
Code: "InvalidTag",
|
||||
Description: "The TagKey you have provided is invalid",
|
||||
Description: "The TagValue you have provided is invalid",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidTagValue: {
|
||||
ErrCode: ErrInvalidTagValue,
|
||||
Code: "InvalidTag",
|
||||
Description: "The TagValue you have provided is invalid",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidTagKeyUniqueness: {
|
||||
ErrCode: ErrInvalidTagKeyUniqueness,
|
||||
Code: "InvalidTag",
|
||||
Description: "Cannot provide multiple Tags with the same key",
|
||||
Description: "The TagKey you have provided is invalid",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidTagsSizeExceed: {
|
||||
|
@ -621,7 +594,7 @@ var errorCodes = errorCodeMap{
|
|||
ErrAuthorizationHeaderMalformed: {
|
||||
ErrCode: ErrAuthorizationHeaderMalformed,
|
||||
Code: "AuthorizationHeaderMalformed",
|
||||
Description: "The authorization header that you provided is not valid.",
|
||||
Description: "The authorization header is malformed; the region is wrong; expecting 'us-east-1'.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMalformedPOSTRequest: {
|
||||
|
@ -666,12 +639,6 @@ var errorCodes = errorCodeMap{
|
|||
Description: "Policy has invalid resource.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMalformedPolicyNotPrincipal: {
|
||||
ErrCode: ErrMalformedPolicyNotPrincipal,
|
||||
Code: "MalformedPolicy",
|
||||
Description: "Allow with NotPrincipal is not allowed.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMissingFields: {
|
||||
ErrCode: ErrMissingFields,
|
||||
Code: "MissingFields",
|
||||
|
@ -1077,12 +1044,6 @@ var errorCodes = errorCodeMap{
|
|||
Description: "Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMissingSSECustomerAlgorithm: {
|
||||
ErrCode: ErrMissingSSECustomerAlgorithm,
|
||||
Code: "InvalidArgument",
|
||||
Description: "Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMissingSSECustomerKeyMD5: {
|
||||
ErrCode: ErrMissingSSECustomerKeyMD5,
|
||||
Code: "InvalidArgument",
|
||||
|
@ -1163,12 +1124,6 @@ var errorCodes = errorCodeMap{
|
|||
Description: "A timeout occurred while trying to lock a resource, please reduce your request rate",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrGatewayTimeout: {
|
||||
ErrCode: ErrGatewayTimeout,
|
||||
Code: "GatewayTimeout",
|
||||
Description: "The server is acting as a gateway and cannot get a response in time",
|
||||
HTTPStatusCode: http.StatusGatewayTimeout,
|
||||
},
|
||||
ErrOperationMaxedOut: {
|
||||
ErrCode: ErrOperationMaxedOut,
|
||||
Code: "SlowDown",
|
||||
|
@ -1197,12 +1152,6 @@ var errorCodes = errorCodeMap{
|
|||
Description: "Invalid Request",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidRequestLargeCopy: {
|
||||
ErrCode: ErrInvalidRequestLargeCopy,
|
||||
Code: "InvalidRequest",
|
||||
Description: "CopyObject request made on objects larger than 5GB in size.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrIncorrectContinuationToken: {
|
||||
ErrCode: ErrIncorrectContinuationToken,
|
||||
Code: "InvalidArgument",
|
||||
|
@ -1744,7 +1693,6 @@ var errorCodes = errorCodeMap{
|
|||
|
||||
// IsS3Error checks if the provided error is a specific s3 error.
|
||||
func IsS3Error(err error, code ErrorCode) bool {
|
||||
err = frosterrors.UnwrapErr(err)
|
||||
e, ok := err.(Error)
|
||||
return ok && e.ErrCode == code
|
||||
}
|
||||
|
|
|
@ -6,28 +6,23 @@ import (
|
|||
"crypto/elliptic"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
v2acl "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
engineiam "git.frostfs.info/TrueCloudLab/policy-engine/iam"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
v2acl "github.com/TrueCloudLab/frostfs-api-go/v2/acl"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -248,8 +243,7 @@ func (s *statement) UnmarshalJSON(data []byte) error {
|
|||
}
|
||||
|
||||
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -257,101 +251,28 @@ func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if bktInfo.APEEnabled || len(settings.CannedACL) != 0 {
|
||||
if err = middleware.EncodeToResponse(w, h.encodeBucketCannedACL(ctx, bktInfo, settings)); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
bucketACL, err := h.obj.GetBucketACL(ctx, bktInfo)
|
||||
bucketACL, err := h.obj.GetBucketACL(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not fetch bucket acl", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, h.encodeBucketACL(ctx, bktInfo.Name, bucketACL)); err != nil {
|
||||
if err = api.EncodeToResponse(w, h.encodeBucketACL(bktInfo.Name, bucketACL)); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) encodeBucketCannedACL(ctx context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) *AccessControlPolicy {
|
||||
res := h.encodePrivateCannedACL(ctx, bktInfo, settings)
|
||||
|
||||
switch settings.CannedACL {
|
||||
case basicACLPublic:
|
||||
grantee := NewGrantee(acpGroup)
|
||||
grantee.URI = allUsersGroup
|
||||
|
||||
res.AccessControlList = append(res.AccessControlList, &Grant{
|
||||
Grantee: grantee,
|
||||
Permission: aclWrite,
|
||||
})
|
||||
fallthrough
|
||||
case basicACLReadOnly:
|
||||
grantee := NewGrantee(acpGroup)
|
||||
grantee.URI = allUsersGroup
|
||||
|
||||
res.AccessControlList = append(res.AccessControlList, &Grant{
|
||||
Grantee: grantee,
|
||||
Permission: aclRead,
|
||||
})
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (h *handler) encodePrivateCannedACL(ctx context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) *AccessControlPolicy {
|
||||
ownerDisplayName := bktInfo.Owner.EncodeToString()
|
||||
ownerEncodedID := ownerDisplayName
|
||||
|
||||
if settings.OwnerKey == nil {
|
||||
h.reqLogger(ctx).Warn(logs.BucketOwnerKeyIsMissing, zap.String("owner", bktInfo.Owner.String()))
|
||||
} else {
|
||||
ownerDisplayName = settings.OwnerKey.Address()
|
||||
ownerEncodedID = hex.EncodeToString(settings.OwnerKey.Bytes())
|
||||
}
|
||||
|
||||
res := &AccessControlPolicy{Owner: Owner{
|
||||
ID: ownerEncodedID,
|
||||
DisplayName: ownerDisplayName,
|
||||
}}
|
||||
|
||||
granteeOwner := NewGrantee(acpCanonicalUser)
|
||||
granteeOwner.ID = ownerEncodedID
|
||||
granteeOwner.DisplayName = ownerDisplayName
|
||||
|
||||
res.AccessControlList = []*Grant{{
|
||||
Grantee: granteeOwner,
|
||||
Permission: aclFullControl,
|
||||
}}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (h *handler) bearerTokenIssuerKey(ctx context.Context) (*keys.PublicKey, error) {
|
||||
box, err := middleware.GetBoxData(ctx)
|
||||
box, err := layer.GetBoxData(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return getTokenIssuerKey(box)
|
||||
}
|
||||
var btoken v2acl.BearerToken
|
||||
box.Gate.BearerToken.WriteToV2(&btoken)
|
||||
|
||||
func getTokenIssuerKey(box *accessbox.Box) (*keys.PublicKey, error) {
|
||||
if box.Gate.BearerToken == nil {
|
||||
return nil, stderrors.New("bearer token is missing")
|
||||
}
|
||||
|
||||
key, err := keys.NewPublicKeyFromBytes(box.Gate.BearerToken.SigningKeyBytes(), elliptic.P256())
|
||||
key, err := keys.NewPublicKeyFromBytes(btoken.GetSignature().GetKey(), elliptic.P256())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("public key from bytes: %w", err)
|
||||
}
|
||||
|
@ -360,25 +281,7 @@ func getTokenIssuerKey(box *accessbox.Box) (*keys.PublicKey, error) {
|
|||
}
|
||||
|
||||
func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if bktInfo.APEEnabled || len(settings.CannedACL) != 0 {
|
||||
h.putBucketACLAPEHandler(w, r, reqInfo, bktInfo, settings)
|
||||
return
|
||||
}
|
||||
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
key, err := h.bearerTokenIssuerKey(r.Context())
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't get bearer token issuer key", reqInfo, err)
|
||||
|
@ -398,8 +301,8 @@ func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
|||
h.logAndSendError(w, "could not parse bucket acl", reqInfo, err)
|
||||
return
|
||||
}
|
||||
} else if err = h.cfg.NewXMLDecoder(r.Body).Decode(list); err != nil {
|
||||
h.logAndSendError(w, "could not parse bucket acl", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
|
||||
} else if err = xml.NewDecoder(r.Body).Decode(list); err != nil {
|
||||
h.logAndSendError(w, "could not parse bucket acl", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -410,6 +313,12 @@ func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err = h.updateBucketACL(r, astBucket, bktInfo, token); err != nil {
|
||||
h.logAndSendError(w, "could not update bucket acl", reqInfo, err)
|
||||
return
|
||||
|
@ -417,60 +326,6 @@ func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
|||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *handler) putBucketACLAPEHandler(w http.ResponseWriter, r *http.Request, reqInfo *middleware.ReqInfo, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
|
||||
ctx := r.Context()
|
||||
|
||||
defer func() {
|
||||
if errBody := r.Body.Close(); errBody != nil {
|
||||
h.reqLogger(r.Context()).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody))
|
||||
}
|
||||
}()
|
||||
|
||||
written, err := io.Copy(io.Discard, r.Body)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't read request body", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if written != 0 || len(r.Header.Get(api.AmzACL)) == 0 {
|
||||
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
||||
return
|
||||
}
|
||||
|
||||
cannedACL, err := parseCannedACL(r.Header)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not parse canned ACL", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
key, err := h.bearerTokenIssuerKey(ctx)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't get bearer token issuer key", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
chainRules := bucketCannedACLToAPERules(cannedACL, reqInfo, key, bktInfo.CID)
|
||||
if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chainRules); err != nil {
|
||||
h.logAndSendError(w, "failed to add morph rule chains", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
settings.CannedACL = cannedACL
|
||||
|
||||
sp := &layer.PutSettingsParams{
|
||||
BktInfo: bktInfo,
|
||||
Settings: settings,
|
||||
}
|
||||
|
||||
if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
|
||||
h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
|
||||
zap.String("container_id", bktInfo.CID.EncodeToString()))
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *handler) updateBucketACL(r *http.Request, astChild *ast, bktInfo *data.BucketInfo, sessionToken *session.Container) (bool, error) {
|
||||
bucketACL, err := h.obj.GetBucketACL(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
|
@ -510,8 +365,7 @@ func (h *handler) updateBucketACL(r *http.Request, astChild *ast, bktInfo *data.
|
|||
}
|
||||
|
||||
func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -519,21 +373,7 @@ func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if bktInfo.APEEnabled || len(settings.CannedACL) != 0 {
|
||||
if err = middleware.EncodeToResponse(w, h.encodePrivateCannedACL(ctx, bktInfo, settings)); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
bucketACL, err := h.obj.GetBucketACL(ctx, bktInfo)
|
||||
bucketACL, err := h.obj.GetBucketACL(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not fetch bucket acl", reqInfo, err)
|
||||
return
|
||||
|
@ -545,63 +385,45 @@ func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
|||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||
}
|
||||
|
||||
objInfo, err := h.obj.GetObjectInfo(ctx, prm)
|
||||
objInfo, err := h.obj.GetObjectInfo(r.Context(), prm)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get object info", reqInfo, err)
|
||||
h.logAndSendError(w, "could not object info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, h.encodeObjectACL(ctx, bucketACL, reqInfo.BucketName, objInfo.VersionID())); err != nil {
|
||||
if err = api.EncodeToResponse(w, h.encodeObjectACL(bucketACL, reqInfo.BucketName, objInfo.VersionID())); err != nil {
|
||||
h.logAndSendError(w, "failed to encode response", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
apeEnabled := bktInfo.APEEnabled
|
||||
|
||||
if !apeEnabled {
|
||||
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
|
||||
return
|
||||
}
|
||||
apeEnabled = len(settings.CannedACL) != 0
|
||||
}
|
||||
|
||||
if apeEnabled {
|
||||
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
||||
return
|
||||
}
|
||||
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
versionID := reqInfo.URL.Query().Get(api.QueryVersionID)
|
||||
key, err := h.bearerTokenIssuerKey(ctx)
|
||||
key, err := h.bearerTokenIssuerKey(r.Context())
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't get gate key", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
token, err := getSessionTokenSetEACL(ctx)
|
||||
token, err := getSessionTokenSetEACL(r.Context())
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't get eacl token", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
p := &layer.HeadObjectParams{
|
||||
BktInfo: bktInfo,
|
||||
Object: reqInfo.ObjectName,
|
||||
VersionID: versionID,
|
||||
}
|
||||
|
||||
objInfo, err := h.obj.GetObjectInfo(ctx, p)
|
||||
objInfo, err := h.obj.GetObjectInfo(r.Context(), p)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get object info", reqInfo, err)
|
||||
return
|
||||
|
@ -614,8 +436,8 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
|||
h.logAndSendError(w, "could not parse bucket acl", reqInfo, err)
|
||||
return
|
||||
}
|
||||
} else if err = h.cfg.NewXMLDecoder(r.Body).Decode(list); err != nil {
|
||||
h.logAndSendError(w, "could not parse bucket acl", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
|
||||
} else if err = xml.NewDecoder(r.Body).Decode(list); err != nil {
|
||||
h.logAndSendError(w, "could not parse bucket acl", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -639,61 +461,19 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
|||
if updated {
|
||||
s := &SendNotificationParams{
|
||||
Event: EventObjectACLPut,
|
||||
NotificationInfo: data.NotificationInfoFromObject(objInfo, h.cfg.MD5Enabled()),
|
||||
NotificationInfo: data.NotificationInfoFromObject(objInfo),
|
||||
BktInfo: bktInfo,
|
||||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
if err = h.sendNotifications(r.Context(), s); err != nil {
|
||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
jsonPolicy, err := h.ape.GetBucketPolicy(reqInfo.Namespace, bktInfo.CID)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
err = fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchBucketPolicy), err.Error())
|
||||
}
|
||||
h.logAndSendError(w, "failed to get policy from storage", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
var bktPolicy engineiam.Policy
|
||||
if err = json.Unmarshal(jsonPolicy, &bktPolicy); err != nil {
|
||||
h.logAndSendError(w, "could not parse bucket policy", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
policyStatus := &PolicyStatus{
|
||||
IsPublic: PolicyStatusIsPublicFalse,
|
||||
}
|
||||
|
||||
for _, st := range bktPolicy.Statement {
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-block-public-access.html#access-control-block-public-access-policy-status
|
||||
if _, ok := st.Principal[engineiam.Wildcard]; ok {
|
||||
policyStatus.IsPublic = PolicyStatusIsPublicTrue
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, policyStatus); err != nil {
|
||||
h.logAndSendError(w, "encode and write response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -701,36 +481,19 @@ func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
|
|||
return
|
||||
}
|
||||
|
||||
jsonPolicy, err := h.ape.GetBucketPolicy(reqInfo.Namespace, bktInfo.CID)
|
||||
bucketACL, err := h.obj.GetBucketACL(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
err = fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchBucketPolicy), err.Error())
|
||||
}
|
||||
h.logAndSendError(w, "failed to get policy from storage", reqInfo, err)
|
||||
h.logAndSendError(w, "could not fetch bucket acl", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set(api.ContentType, "application/json")
|
||||
ast := tableToAst(bucketACL.EACL, reqInfo.BucketName)
|
||||
bktPolicy := astToPolicy(ast)
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
if _, err = w.Write(jsonPolicy); err != nil {
|
||||
h.logAndSendError(w, "write json policy to client", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
chainIDs := []chain.ID{getBucketChainID(chain.S3, bktInfo), getBucketChainID(chain.Ingress, bktInfo)}
|
||||
if err = h.ape.DeleteBucketPolicy(reqInfo.Namespace, bktInfo.CID, chainIDs); err != nil {
|
||||
h.logAndSendError(w, "failed to delete policy from storage", reqInfo, err)
|
||||
return
|
||||
if err = json.NewEncoder(w).Encode(bktPolicy); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -741,14 +504,14 @@ func checkOwner(info *data.BucketInfo, owner string) error {
|
|||
|
||||
// may need to convert owner to appropriate format
|
||||
if info.Owner.String() != owner {
|
||||
return fmt.Errorf("%w: mismatch owner", errors.GetAPIError(errors.ErrAccessDenied))
|
||||
return errors.GetAPIError(errors.ErrAccessDenied)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -756,91 +519,30 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
|
|||
return
|
||||
}
|
||||
|
||||
jsonPolicy, err := io.ReadAll(r.Body)
|
||||
token, err := getSessionTokenSetEACL(r.Context())
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "read body", reqInfo, err)
|
||||
h.logAndSendError(w, "couldn't get eacl token", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
var bktPolicy engineiam.Policy
|
||||
if err = json.Unmarshal(jsonPolicy, &bktPolicy); err != nil {
|
||||
bktPolicy := &bucketPolicy{Bucket: reqInfo.BucketName}
|
||||
if err = json.NewDecoder(r.Body).Decode(bktPolicy); err != nil {
|
||||
h.logAndSendError(w, "could not parse bucket policy", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, stat := range bktPolicy.Statement {
|
||||
if len(stat.NotResource) != 0 {
|
||||
h.logAndSendError(w, "policy resource mismatched bucket", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicy))
|
||||
return
|
||||
}
|
||||
|
||||
if len(stat.NotPrincipal) != 0 && stat.Effect == engineiam.AllowEffect {
|
||||
h.logAndSendError(w, "invalid NotPrincipal", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicyNotPrincipal))
|
||||
return
|
||||
}
|
||||
|
||||
for _, resource := range stat.Resource {
|
||||
if reqInfo.BucketName != strings.Split(strings.TrimPrefix(resource, arnAwsPrefix), "/")[0] {
|
||||
h.logAndSendError(w, "policy resource mismatched bucket", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicy))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
s3Chain, err := engineiam.ConvertToS3Chain(bktPolicy, h.frostfsid)
|
||||
astPolicy, err := policyToAst(bktPolicy)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not convert s3 policy to chain policy", reqInfo, err)
|
||||
h.logAndSendError(w, "could not translate policy to ast", reqInfo, err)
|
||||
return
|
||||
}
|
||||
s3Chain.ID = getBucketChainID(chain.S3, bktInfo)
|
||||
|
||||
nativeChain, err := engineiam.ConvertToNativeChain(bktPolicy, h.nativeResolver(reqInfo.Namespace, bktInfo))
|
||||
if err == nil {
|
||||
nativeChain.ID = getBucketChainID(chain.Ingress, bktInfo)
|
||||
} else if !stderrors.Is(err, engineiam.ErrActionsNotApplicable) {
|
||||
h.logAndSendError(w, "could not convert s3 policy to native chain policy", reqInfo, err)
|
||||
return
|
||||
} else {
|
||||
h.reqLogger(r.Context()).Warn(logs.PolicyCouldntBeConvertedToNativeRules)
|
||||
}
|
||||
|
||||
chainsToSave := []*chain.Chain{s3Chain}
|
||||
if nativeChain != nil {
|
||||
chainsToSave = append(chainsToSave, nativeChain)
|
||||
}
|
||||
|
||||
if err = h.ape.PutBucketPolicy(reqInfo.Namespace, bktInfo.CID, jsonPolicy, chainsToSave); err != nil {
|
||||
h.logAndSendError(w, "failed to update policy in contract", reqInfo, err)
|
||||
if _, err = h.updateBucketACL(r, astPolicy, bktInfo, token); err != nil {
|
||||
h.logAndSendError(w, "could not update bucket acl", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
type nativeResolver struct {
|
||||
FrostFSID
|
||||
namespace string
|
||||
bktInfo *data.BucketInfo
|
||||
}
|
||||
|
||||
func (n *nativeResolver) GetBucketInfo(bucket string) (*engineiam.BucketInfo, error) {
|
||||
if n.bktInfo.Name != bucket {
|
||||
return nil, fmt.Errorf("invalid bucket %s: %w", bucket, errors.GetAPIError(errors.ErrMalformedPolicy))
|
||||
}
|
||||
|
||||
return &engineiam.BucketInfo{Namespace: n.namespace, Container: n.bktInfo.CID.EncodeToString()}, nil
|
||||
}
|
||||
|
||||
func (h *handler) nativeResolver(ns string, bktInfo *data.BucketInfo) engineiam.NativeResolver {
|
||||
return &nativeResolver{
|
||||
FrostFSID: h.frostfsid,
|
||||
namespace: ns,
|
||||
bktInfo: bktInfo,
|
||||
}
|
||||
}
|
||||
|
||||
func getBucketChainID(prefix chain.Name, bktInfo *data.BucketInfo) chain.ID {
|
||||
return chain.ID(string(prefix) + ":bkt" + string(bktInfo.CID[:]))
|
||||
}
|
||||
|
||||
func parseACLHeaders(header http.Header, key *keys.PublicKey) (*AccessControlPolicy, error) {
|
||||
var err error
|
||||
acp := &AccessControlPolicy{Owner: Owner{
|
||||
|
@ -936,7 +638,7 @@ func parseGrantee(grantees string) ([]*Grantee, error) {
|
|||
}
|
||||
|
||||
func formGrantee(granteeType, value string) (*Grantee, error) {
|
||||
value = data.UnQuote(value)
|
||||
value = strings.Trim(value, "\"")
|
||||
switch granteeType {
|
||||
case "id":
|
||||
return &Grantee{
|
||||
|
@ -1428,6 +1130,73 @@ func resourceInfoFromName(name, bucketName string) resourceInfo {
|
|||
return resInfo
|
||||
}
|
||||
|
||||
func astToPolicy(ast *ast) *bucketPolicy {
|
||||
bktPolicy := &bucketPolicy{}
|
||||
|
||||
for _, resource := range ast.Resources {
|
||||
allowed, denied := triageOperations(resource.Operations)
|
||||
handleResourceOperations(bktPolicy, allowed, eacl.ActionAllow, resource.Name())
|
||||
handleResourceOperations(bktPolicy, denied, eacl.ActionDeny, resource.Name())
|
||||
}
|
||||
|
||||
return bktPolicy
|
||||
}
|
||||
|
||||
func handleResourceOperations(bktPolicy *bucketPolicy, list []*astOperation, eaclAction eacl.Action, resourceName string) {
|
||||
userOpsMap := make(map[string][]eacl.Operation)
|
||||
|
||||
for _, op := range list {
|
||||
if !op.IsGroupGrantee() {
|
||||
for _, user := range op.Users {
|
||||
userOps := userOpsMap[user]
|
||||
userOps = append(userOps, op.Op)
|
||||
userOpsMap[user] = userOps
|
||||
}
|
||||
} else {
|
||||
userOps := userOpsMap[allUsersGroup]
|
||||
userOps = append(userOps, op.Op)
|
||||
userOpsMap[allUsersGroup] = userOps
|
||||
}
|
||||
}
|
||||
|
||||
for user, userOps := range userOpsMap {
|
||||
var actions []string
|
||||
LOOP:
|
||||
for action, ops := range actionToOpMap {
|
||||
for _, op := range ops {
|
||||
if !contains(userOps, op) {
|
||||
continue LOOP
|
||||
}
|
||||
}
|
||||
actions = append(actions, action)
|
||||
}
|
||||
if len(actions) != 0 {
|
||||
state := statement{
|
||||
Effect: actionToEffect(eaclAction),
|
||||
Principal: principal{CanonicalUser: user},
|
||||
Action: actions,
|
||||
Resource: []string{arnAwsPrefix + resourceName},
|
||||
}
|
||||
if user == allUsersGroup {
|
||||
state.Principal = principal{AWS: allUsersWildcard}
|
||||
}
|
||||
bktPolicy.Statement = append(bktPolicy.Statement, state)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func triageOperations(operations []*astOperation) ([]*astOperation, []*astOperation) {
|
||||
var allowed, denied []*astOperation
|
||||
for _, op := range operations {
|
||||
if op.Action == eacl.ActionAllow {
|
||||
allowed = append(allowed, op)
|
||||
} else {
|
||||
denied = append(denied, op)
|
||||
}
|
||||
}
|
||||
return allowed, denied
|
||||
}
|
||||
|
||||
func addTo(list []*astOperation, userID string, op eacl.Operation, groupGrantee bool, action eacl.Action) []*astOperation {
|
||||
var found *astOperation
|
||||
for _, astop := range list {
|
||||
|
@ -1614,6 +1383,17 @@ func effectToAction(effect string) eacl.Action {
|
|||
return eacl.ActionUnknown
|
||||
}
|
||||
|
||||
func actionToEffect(action eacl.Action) string {
|
||||
switch action {
|
||||
case eacl.ActionAllow:
|
||||
return "Allow"
|
||||
case eacl.ActionDeny:
|
||||
return "Deny"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func permissionToOperations(permission AWSACL) []eacl.Operation {
|
||||
switch permission {
|
||||
case aclFullControl:
|
||||
|
@ -1630,27 +1410,7 @@ func isWriteOperation(op eacl.Operation) bool {
|
|||
return op == eacl.OperationDelete || op == eacl.OperationPut
|
||||
}
|
||||
|
||||
type access struct {
|
||||
recipient string
|
||||
operations []eacl.Operation
|
||||
}
|
||||
|
||||
type accessList struct {
|
||||
list []access
|
||||
}
|
||||
|
||||
func (c *accessList) addAccess(recipient string, operation eacl.Operation) {
|
||||
for i, v := range c.list {
|
||||
if v.recipient == recipient {
|
||||
c.list[i].operations = append(c.list[i].operations, operation)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
c.list = append(c.list, access{recipient, []eacl.Operation{operation}})
|
||||
}
|
||||
|
||||
func (h *handler) encodeObjectACL(ctx context.Context, bucketACL *layer.BucketACL, bucketName, objectVersion string) *AccessControlPolicy {
|
||||
func (h *handler) encodeObjectACL(bucketACL *layer.BucketACL, bucketName, objectVersion string) *AccessControlPolicy {
|
||||
res := &AccessControlPolicy{
|
||||
Owner: Owner{
|
||||
ID: bucketACL.Info.Owner.String(),
|
||||
|
@ -1658,7 +1418,7 @@ func (h *handler) encodeObjectACL(ctx context.Context, bucketACL *layer.BucketAC
|
|||
},
|
||||
}
|
||||
|
||||
m := &accessList{}
|
||||
m := make(map[string][]eacl.Operation)
|
||||
|
||||
astList := tableToAst(bucketACL.EACL, bucketName)
|
||||
|
||||
|
@ -1673,20 +1433,22 @@ func (h *handler) encodeObjectACL(ctx context.Context, bucketACL *layer.BucketAC
|
|||
}
|
||||
|
||||
if len(op.Users) == 0 {
|
||||
m.addAccess(allUsersGroup, op.Op)
|
||||
list := append(m[allUsersGroup], op.Op)
|
||||
m[allUsersGroup] = list
|
||||
} else {
|
||||
for _, user := range op.Users {
|
||||
m.addAccess(user, op.Op)
|
||||
list := append(m[user], op.Op)
|
||||
m[user] = list
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, val := range m.list {
|
||||
for key, val := range m {
|
||||
permission := aclFullControl
|
||||
read := true
|
||||
for op := eacl.OperationGet; op <= eacl.OperationRangeHash; op++ {
|
||||
if !contains(val.operations, op) && !isWriteOperation(op) {
|
||||
if !contains(val, op) && !isWriteOperation(op) {
|
||||
read = false
|
||||
}
|
||||
}
|
||||
|
@ -1694,16 +1456,16 @@ func (h *handler) encodeObjectACL(ctx context.Context, bucketACL *layer.BucketAC
|
|||
if read {
|
||||
permission = aclFullControl
|
||||
} else {
|
||||
h.reqLogger(ctx).Warn(logs.SomeACLNotFullyMapped)
|
||||
h.log.Warn("some acl not fully mapped")
|
||||
}
|
||||
|
||||
var grantee *Grantee
|
||||
if val.recipient == allUsersGroup {
|
||||
if key == allUsersGroup {
|
||||
grantee = NewGrantee(acpGroup)
|
||||
grantee.URI = allUsersGroup
|
||||
} else {
|
||||
grantee = NewGrantee(acpCanonicalUser)
|
||||
grantee.ID = val.recipient
|
||||
grantee.ID = key
|
||||
}
|
||||
|
||||
grant := &Grant{
|
||||
|
@ -1716,8 +1478,8 @@ func (h *handler) encodeObjectACL(ctx context.Context, bucketACL *layer.BucketAC
|
|||
return res
|
||||
}
|
||||
|
||||
func (h *handler) encodeBucketACL(ctx context.Context, bucketName string, bucketACL *layer.BucketACL) *AccessControlPolicy {
|
||||
return h.encodeObjectACL(ctx, bucketACL, bucketName, "")
|
||||
func (h *handler) encodeBucketACL(bucketName string, bucketACL *layer.BucketACL) *AccessControlPolicy {
|
||||
return h.encodeObjectACL(bucketACL, bucketName, "")
|
||||
}
|
||||
|
||||
func contains(list []eacl.Operation, op eacl.Operation) bool {
|
||||
|
|
|
@ -2,31 +2,25 @@ package handler
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
engineiam "git.frostfs.info/TrueCloudLab/policy-engine/iam"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
@ -1303,235 +1297,59 @@ func TestBucketAclToAst(t *testing.T) {
|
|||
|
||||
func TestPutBucketACL(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
tc.config.aclEnabled = true
|
||||
bktName := "bucket-for-acl"
|
||||
|
||||
info := createBucket(tc, bktName)
|
||||
box, _ := createAccessBox(t)
|
||||
bktInfo := createBucket(t, tc, bktName, box)
|
||||
|
||||
header := map[string]string{api.AmzACL: "public-read"}
|
||||
putBucketACL(tc, bktName, info.Box, header)
|
||||
putBucketACL(t, tc, bktName, box, header)
|
||||
|
||||
header = map[string]string{api.AmzACL: "private"}
|
||||
putBucketACL(tc, bktName, info.Box, header)
|
||||
checkLastRecords(t, tc, info.BktInfo, eacl.ActionDeny)
|
||||
}
|
||||
|
||||
func TestPutBucketAPE(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName := "bucket-for-acl-ape"
|
||||
|
||||
info := createBucket(hc, bktName)
|
||||
|
||||
_, err := hc.tp.ContainerEACL(hc.Context(), layer.PrmContainerEACL{ContainerID: info.BktInfo.CID})
|
||||
require.ErrorContains(t, err, "not found")
|
||||
|
||||
chains, err := hc.h.ape.(*apeMock).ListChains(engine.ContainerTarget(info.BktInfo.CID.EncodeToString()))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, chains, 2)
|
||||
}
|
||||
|
||||
func TestPutObjectACLErrorAPE(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName, objName := "bucket-for-acl-ape", "object"
|
||||
|
||||
info := createBucket(hc, bktName)
|
||||
|
||||
putObjectWithHeadersAssertS3Error(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPublic}, s3errors.ErrAccessControlListNotSupported)
|
||||
putObjectWithHeaders(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPrivate}) // only `private` canned acl is allowed, that is actually ignored
|
||||
putObjectWithHeaders(hc, bktName, objName, nil)
|
||||
|
||||
aclBody := &AccessControlPolicy{}
|
||||
putObjectACLAssertS3Error(hc, bktName, objName, info.Box, nil, aclBody, s3errors.ErrAccessControlListNotSupported)
|
||||
|
||||
aclRes := getObjectACL(hc, bktName, objName)
|
||||
checkPrivateACL(t, aclRes, info.Key.PublicKey())
|
||||
}
|
||||
|
||||
func TestCreateObjectACLErrorAPE(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName, objName, objNameCopy := "bucket-for-acl-ape", "object", "copy"
|
||||
|
||||
createBucket(hc, bktName)
|
||||
|
||||
putObject(hc, bktName, objName)
|
||||
copyObject(hc, bktName, objName, objNameCopy, CopyMeta{Headers: map[string]string{api.AmzACL: basicACLPublic}}, http.StatusBadRequest)
|
||||
copyObject(hc, bktName, objName, objNameCopy, CopyMeta{Headers: map[string]string{api.AmzACL: basicACLPrivate}}, http.StatusOK)
|
||||
|
||||
createMultipartUploadAssertS3Error(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPublic}, s3errors.ErrAccessControlListNotSupported)
|
||||
createMultipartUpload(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPrivate})
|
||||
}
|
||||
|
||||
func TestPutObjectACLBackwardCompatibility(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
hc.config.aclEnabled = true
|
||||
bktName, objName := "bucket-for-acl-ape", "object"
|
||||
|
||||
info := createBucket(hc, bktName)
|
||||
|
||||
putObjectWithHeadersBase(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPrivate}, info.Box, nil)
|
||||
putObjectWithHeadersBase(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPublic}, info.Box, nil)
|
||||
|
||||
aclRes := getObjectACL(hc, bktName, objName)
|
||||
require.Len(t, aclRes.AccessControlList, 2)
|
||||
require.Equal(t, hex.EncodeToString(info.Key.PublicKey().Bytes()), aclRes.AccessControlList[0].Grantee.ID)
|
||||
require.Equal(t, aclFullControl, aclRes.AccessControlList[0].Permission)
|
||||
require.Equal(t, allUsersGroup, aclRes.AccessControlList[1].Grantee.URI)
|
||||
require.Equal(t, aclFullControl, aclRes.AccessControlList[1].Permission)
|
||||
|
||||
aclBody := &AccessControlPolicy{}
|
||||
putObjectACLBase(hc, bktName, objName, info.Box, nil, aclBody)
|
||||
}
|
||||
|
||||
func TestBucketACLAPE(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName := "bucket-for-acl-ape"
|
||||
|
||||
info := createBucket(hc, bktName)
|
||||
|
||||
aclBody := &AccessControlPolicy{}
|
||||
putBucketACLAssertS3Error(hc, bktName, info.Box, nil, aclBody, s3errors.ErrAccessControlListNotSupported)
|
||||
|
||||
aclRes := getBucketACL(hc, bktName)
|
||||
checkPrivateACL(t, aclRes, info.Key.PublicKey())
|
||||
|
||||
putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLPrivate})
|
||||
aclRes = getBucketACL(hc, bktName)
|
||||
checkPrivateACL(t, aclRes, info.Key.PublicKey())
|
||||
|
||||
putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLReadOnly})
|
||||
aclRes = getBucketACL(hc, bktName)
|
||||
checkPublicReadACL(t, aclRes, info.Key.PublicKey())
|
||||
|
||||
putBucketACL(hc, bktName, info.Box, map[string]string{api.AmzACL: basicACLPublic})
|
||||
aclRes = getBucketACL(hc, bktName)
|
||||
checkPublicReadWriteACL(t, aclRes, info.Key.PublicKey())
|
||||
}
|
||||
|
||||
func checkPrivateACL(t *testing.T, aclRes *AccessControlPolicy, ownerKey *keys.PublicKey) {
|
||||
checkACLOwner(t, aclRes, ownerKey, 1)
|
||||
}
|
||||
|
||||
func checkPublicReadACL(t *testing.T, aclRes *AccessControlPolicy, ownerKey *keys.PublicKey) {
|
||||
checkACLOwner(t, aclRes, ownerKey, 2)
|
||||
|
||||
require.Equal(t, allUsersGroup, aclRes.AccessControlList[1].Grantee.URI)
|
||||
require.Equal(t, aclRead, aclRes.AccessControlList[1].Permission)
|
||||
}
|
||||
|
||||
func checkPublicReadWriteACL(t *testing.T, aclRes *AccessControlPolicy, ownerKey *keys.PublicKey) {
|
||||
checkACLOwner(t, aclRes, ownerKey, 3)
|
||||
|
||||
require.Equal(t, allUsersGroup, aclRes.AccessControlList[1].Grantee.URI)
|
||||
require.Equal(t, aclWrite, aclRes.AccessControlList[1].Permission)
|
||||
|
||||
require.Equal(t, allUsersGroup, aclRes.AccessControlList[2].Grantee.URI)
|
||||
require.Equal(t, aclRead, aclRes.AccessControlList[2].Permission)
|
||||
}
|
||||
|
||||
func checkACLOwner(t *testing.T, aclRes *AccessControlPolicy, ownerKey *keys.PublicKey, ln int) {
|
||||
ownerIDStr := hex.EncodeToString(ownerKey.Bytes())
|
||||
ownerNameStr := ownerKey.Address()
|
||||
|
||||
require.Equal(t, ownerIDStr, aclRes.Owner.ID)
|
||||
require.Equal(t, ownerNameStr, aclRes.Owner.DisplayName)
|
||||
|
||||
require.Len(t, aclRes.AccessControlList, ln)
|
||||
|
||||
require.Equal(t, ownerIDStr, aclRes.AccessControlList[0].Grantee.ID)
|
||||
require.Equal(t, ownerNameStr, aclRes.AccessControlList[0].Grantee.DisplayName)
|
||||
require.Equal(t, aclFullControl, aclRes.AccessControlList[0].Permission)
|
||||
putBucketACL(t, tc, bktName, box, header)
|
||||
checkLastRecords(t, tc, bktInfo, eacl.ActionDeny)
|
||||
}
|
||||
|
||||
func TestBucketPolicy(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName := "bucket-for-policy"
|
||||
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
getBucketPolicy(hc, bktName, s3errors.ErrNoSuchBucketPolicy)
|
||||
|
||||
newPolicy := engineiam.Policy{
|
||||
Version: "2012-10-17",
|
||||
Statement: []engineiam.Statement{{
|
||||
Principal: map[engineiam.PrincipalType][]string{engineiam.Wildcard: {}},
|
||||
Effect: engineiam.DenyEffect,
|
||||
Action: engineiam.Action{"s3:PutObject"},
|
||||
Resource: engineiam.Resource{"arn:aws:s3:::test/*"},
|
||||
}},
|
||||
}
|
||||
|
||||
putBucketPolicy(hc, bktName, newPolicy, s3errors.ErrMalformedPolicy)
|
||||
|
||||
newPolicy.Statement[0].Resource[0] = arnAwsPrefix + bktName + "/*"
|
||||
putBucketPolicy(hc, bktName, newPolicy)
|
||||
box, key := createAccessBox(t)
|
||||
createBucket(t, hc, bktName, box)
|
||||
|
||||
bktPolicy := getBucketPolicy(hc, bktName)
|
||||
require.Equal(t, newPolicy, bktPolicy)
|
||||
}
|
||||
for _, st := range bktPolicy.Statement {
|
||||
if st.Effect == "Allow" {
|
||||
require.Equal(t, hex.EncodeToString(key.PublicKey().Bytes()), st.Principal.CanonicalUser)
|
||||
require.Equal(t, []string{arnAwsPrefix + bktName}, st.Resource)
|
||||
} else {
|
||||
require.Equal(t, allUsersWildcard, st.Principal.AWS)
|
||||
require.Equal(t, "Deny", st.Effect)
|
||||
require.Equal(t, []string{arnAwsPrefix + bktName}, st.Resource)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketPolicyStatus(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName := "bucket-for-policy"
|
||||
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
getBucketPolicy(hc, bktName, s3errors.ErrNoSuchBucketPolicy)
|
||||
|
||||
newPolicy := engineiam.Policy{
|
||||
Version: "2012-10-17",
|
||||
Statement: []engineiam.Statement{{
|
||||
NotPrincipal: engineiam.Principal{engineiam.Wildcard: {}},
|
||||
Effect: engineiam.AllowEffect,
|
||||
Action: engineiam.Action{"s3:PutObject"},
|
||||
Resource: engineiam.Resource{arnAwsPrefix + bktName + "/*"},
|
||||
newPolicy := &bucketPolicy{
|
||||
Statement: []statement{{
|
||||
Effect: "Allow",
|
||||
Principal: principal{AWS: allUsersWildcard},
|
||||
Action: []string{s3GetObject},
|
||||
Resource: []string{arnAwsPrefix + "dummy"},
|
||||
}},
|
||||
}
|
||||
|
||||
putBucketPolicy(hc, bktName, newPolicy, s3errors.ErrMalformedPolicyNotPrincipal)
|
||||
putBucketPolicy(hc, bktName, newPolicy, box, http.StatusInternalServerError)
|
||||
|
||||
newPolicy.Statement[0].NotPrincipal = nil
|
||||
newPolicy.Statement[0].Principal = map[engineiam.PrincipalType][]string{engineiam.Wildcard: {}}
|
||||
putBucketPolicy(hc, bktName, newPolicy)
|
||||
bktPolicyStatus := getBucketPolicyStatus(hc, bktName)
|
||||
require.True(t, PolicyStatusIsPublicTrue == bktPolicyStatus.IsPublic)
|
||||
newPolicy.Statement[0].Resource[0] = arnAwsPrefix + bktName
|
||||
putBucketPolicy(hc, bktName, newPolicy, box, http.StatusOK)
|
||||
|
||||
key, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
hc.Handler().frostfsid.(*frostfsidMock).data["devenv"] = key.PublicKey()
|
||||
|
||||
newPolicy.Statement[0].Principal = map[engineiam.PrincipalType][]string{engineiam.AWSPrincipalType: {"arn:aws:iam:::user/devenv"}}
|
||||
putBucketPolicy(hc, bktName, newPolicy)
|
||||
bktPolicyStatus = getBucketPolicyStatus(hc, bktName)
|
||||
require.True(t, PolicyStatusIsPublicFalse == bktPolicyStatus.IsPublic)
|
||||
}
|
||||
|
||||
func TestDeleteBucketWithPolicy(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-policy"
|
||||
bi := createTestBucket(hc, bktName)
|
||||
|
||||
newPolicy := engineiam.Policy{
|
||||
Version: "2012-10-17",
|
||||
Statement: []engineiam.Statement{{
|
||||
Principal: map[engineiam.PrincipalType][]string{engineiam.Wildcard: {}},
|
||||
Effect: engineiam.AllowEffect,
|
||||
Action: engineiam.Action{"s3:PutObject"},
|
||||
Resource: engineiam.Resource{"arn:aws:s3:::bucket-for-policy/*"},
|
||||
}},
|
||||
bktPolicy = getBucketPolicy(hc, bktName)
|
||||
for _, st := range bktPolicy.Statement {
|
||||
if st.Effect == "Allow" && st.Principal.AWS == allUsersWildcard {
|
||||
require.Equal(t, []string{arnAwsPrefix + bktName}, st.Resource)
|
||||
require.ElementsMatch(t, []string{s3GetObject, s3ListBucket}, st.Action)
|
||||
}
|
||||
}
|
||||
|
||||
putBucketPolicy(hc, bktName, newPolicy)
|
||||
|
||||
require.Len(t, hc.h.ape.(*apeMock).policyMap, 1)
|
||||
require.Len(t, hc.h.ape.(*apeMock).chainMap[engine.ContainerTarget(bi.CID.EncodeToString())], 4)
|
||||
|
||||
deleteBucket(t, hc, bktName, http.StatusNoContent)
|
||||
|
||||
require.Empty(t, hc.h.ape.(*apeMock).policyMap)
|
||||
chains, err := hc.h.ape.(*apeMock).ListChains(engine.ContainerTarget(bi.CID.EncodeToString()))
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, chains)
|
||||
}
|
||||
|
||||
func TestBucketPolicyUnmarshal(t *testing.T) {
|
||||
|
@ -1550,7 +1368,7 @@ func TestBucketPolicyUnmarshal(t *testing.T) {
|
|||
},
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:GetObject",
|
||||
"s3:GetObjectVersion"
|
||||
],
|
||||
"Resource": [
|
||||
|
@ -1591,7 +1409,9 @@ func TestPutBucketPolicy(t *testing.T) {
|
|||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Principal": "*",
|
||||
"Principal": {
|
||||
"AWS": "*"
|
||||
},
|
||||
"Effect": "Deny",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::bucket-for-policy/*"
|
||||
|
@ -1601,57 +1421,36 @@ func TestPutBucketPolicy(t *testing.T) {
|
|||
hc := prepareHandlerContext(t)
|
||||
bktName := "bucket-for-policy"
|
||||
|
||||
createTestBucket(hc, bktName)
|
||||
box, _ := createAccessBox(t)
|
||||
createBucket(t, hc, bktName, box)
|
||||
|
||||
w, r := prepareTestPayloadRequest(hc, bktName, "", bytes.NewReader([]byte(bktPolicy)))
|
||||
ctx := context.WithValue(r.Context(), api.BoxData, box)
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().PutBucketPolicyHandler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func getBucketPolicy(hc *handlerContext, bktName string, errCode ...s3errors.ErrorCode) engineiam.Policy {
|
||||
func getBucketPolicy(hc *handlerContext, bktName string) *bucketPolicy {
|
||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||
hc.Handler().GetBucketPolicyHandler(w, r)
|
||||
|
||||
var policy engineiam.Policy
|
||||
if len(errCode) == 0 {
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
err := json.NewDecoder(w.Result().Body).Decode(&policy)
|
||||
require.NoError(hc.t, err)
|
||||
} else {
|
||||
assertS3Error(hc.t, w, s3errors.GetAPIError(errCode[0]))
|
||||
}
|
||||
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
policy := &bucketPolicy{}
|
||||
err := json.NewDecoder(w.Result().Body).Decode(policy)
|
||||
require.NoError(hc.t, err)
|
||||
return policy
|
||||
}
|
||||
|
||||
func getBucketPolicyStatus(hc *handlerContext, bktName string, errCode ...s3errors.ErrorCode) PolicyStatus {
|
||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||
hc.Handler().GetBucketPolicyStatusHandler(w, r)
|
||||
|
||||
var policyStatus PolicyStatus
|
||||
if len(errCode) == 0 {
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
err := xml.NewDecoder(w.Result().Body).Decode(&policyStatus)
|
||||
require.NoError(hc.t, err)
|
||||
} else {
|
||||
assertS3Error(hc.t, w, s3errors.GetAPIError(errCode[0]))
|
||||
}
|
||||
|
||||
return policyStatus
|
||||
}
|
||||
|
||||
func putBucketPolicy(hc *handlerContext, bktName string, bktPolicy engineiam.Policy, errCode ...s3errors.ErrorCode) {
|
||||
func putBucketPolicy(hc *handlerContext, bktName string, bktPolicy *bucketPolicy, box *accessbox.Box, status int) {
|
||||
body, err := json.Marshal(bktPolicy)
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
w, r := prepareTestPayloadRequest(hc, bktName, "", bytes.NewReader(body))
|
||||
ctx := context.WithValue(r.Context(), api.BoxData, box)
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().PutBucketPolicyHandler(w, r)
|
||||
|
||||
if len(errCode) == 0 {
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
} else {
|
||||
assertS3Error(hc.t, w, s3errors.GetAPIError(errCode[0]))
|
||||
}
|
||||
assertStatus(hc.t, w, status)
|
||||
}
|
||||
|
||||
func checkLastRecords(t *testing.T, tc *handlerContext, bktInfo *data.BucketInfo, action eacl.Action) {
|
||||
|
@ -1681,14 +1480,9 @@ func createAccessBox(t *testing.T) (*accessbox.Box, *keys.PrivateKey) {
|
|||
|
||||
tok := new(session.Container)
|
||||
tok.ForVerb(session.VerbContainerSetEACL)
|
||||
err = tok.Sign(key.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
tok2 := new(session.Container)
|
||||
tok2.ForVerb(session.VerbContainerPut)
|
||||
err = tok2.Sign(key.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
box := &accessbox.Box{
|
||||
Gate: &accessbox.GateData{
|
||||
SessionTokens: []*session.Container{tok, tok2},
|
||||
|
@ -1699,128 +1493,25 @@ func createAccessBox(t *testing.T) (*accessbox.Box, *keys.PrivateKey) {
|
|||
return box, key
|
||||
}
|
||||
|
||||
type createBucketInfo struct {
|
||||
BktInfo *data.BucketInfo
|
||||
Box *accessbox.Box
|
||||
Key *keys.PrivateKey
|
||||
}
|
||||
|
||||
func createBucket(hc *handlerContext, bktName string) *createBucketInfo {
|
||||
box, key := createAccessBox(hc.t)
|
||||
|
||||
w := createBucketBase(hc, bktName, box)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
|
||||
bktInfo, err := hc.Layer().GetBucketInfo(hc.Context(), bktName)
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
return &createBucketInfo{
|
||||
BktInfo: bktInfo,
|
||||
Box: box,
|
||||
Key: key,
|
||||
}
|
||||
}
|
||||
|
||||
func createBucketAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, code s3errors.ErrorCode) {
|
||||
w := createBucketBase(hc, bktName, box)
|
||||
assertS3Error(hc.t, w, s3errors.GetAPIError(code))
|
||||
}
|
||||
|
||||
func createBucketBase(hc *handlerContext, bktName string, box *accessbox.Box) *httptest.ResponseRecorder {
|
||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
||||
func createBucket(t *testing.T, tc *handlerContext, bktName string, box *accessbox.Box) *data.BucketInfo {
|
||||
w, r := prepareTestRequest(tc, bktName, "", nil)
|
||||
ctx := context.WithValue(r.Context(), api.BoxData, box)
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().CreateBucketHandler(w, r)
|
||||
return w
|
||||
tc.Handler().CreateBucketHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
bktInfo, err := tc.Layer().GetBucketInfo(tc.Context(), bktName)
|
||||
require.NoError(t, err)
|
||||
return bktInfo
|
||||
}
|
||||
|
||||
func putBucketACL(hc *handlerContext, bktName string, box *accessbox.Box, header map[string]string) {
|
||||
w := putBucketACLBase(hc, bktName, box, header, nil)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func putBucketACLAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy, code s3errors.ErrorCode) {
|
||||
w := putBucketACLBase(hc, bktName, box, header, body)
|
||||
assertS3Error(hc.t, w, s3errors.GetAPIError(code))
|
||||
}
|
||||
|
||||
func putBucketACLBase(hc *handlerContext, bktName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy) *httptest.ResponseRecorder {
|
||||
w, r := prepareTestRequest(hc, bktName, "", body)
|
||||
func putBucketACL(t *testing.T, tc *handlerContext, bktName string, box *accessbox.Box, header map[string]string) {
|
||||
w, r := prepareTestRequest(tc, bktName, "", nil)
|
||||
for key, val := range header {
|
||||
r.Header.Set(key, val)
|
||||
}
|
||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
||||
ctx := context.WithValue(r.Context(), api.BoxData, box)
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().PutBucketACLHandler(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
func getBucketACL(hc *handlerContext, bktName string) *AccessControlPolicy {
|
||||
w := getBucketACLBase(hc, bktName)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
res := &AccessControlPolicy{}
|
||||
parseTestResponse(hc.t, w, res)
|
||||
return res
|
||||
}
|
||||
|
||||
func getBucketACLBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
|
||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||
hc.Handler().GetBucketACLHandler(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
func putObjectACLAssertS3Error(hc *handlerContext, bktName, objName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy, code s3errors.ErrorCode) {
|
||||
w := putObjectACLBase(hc, bktName, objName, box, header, body)
|
||||
assertS3Error(hc.t, w, s3errors.GetAPIError(code))
|
||||
}
|
||||
|
||||
func putObjectACLBase(hc *handlerContext, bktName, objName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy) *httptest.ResponseRecorder {
|
||||
w, r := prepareTestRequest(hc, bktName, objName, body)
|
||||
for key, val := range header {
|
||||
r.Header.Set(key, val)
|
||||
}
|
||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().PutObjectACLHandler(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
func getObjectACL(hc *handlerContext, bktName, objName string) *AccessControlPolicy {
|
||||
w := getObjectACLBase(hc, bktName, objName)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
res := &AccessControlPolicy{}
|
||||
parseTestResponse(hc.t, w, res)
|
||||
return res
|
||||
}
|
||||
|
||||
func getObjectACLBase(hc *handlerContext, bktName, objName string) *httptest.ResponseRecorder {
|
||||
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
||||
hc.Handler().GetObjectACLHandler(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
func putObjectWithHeaders(hc *handlerContext, bktName, objName string, headers map[string]string) http.Header {
|
||||
w := putObjectWithHeadersBase(hc, bktName, objName, headers, nil, nil)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
return w.Header()
|
||||
}
|
||||
|
||||
func putObjectWithHeadersAssertS3Error(hc *handlerContext, bktName, objName string, headers map[string]string, code s3errors.ErrorCode) {
|
||||
w := putObjectWithHeadersBase(hc, bktName, objName, headers, nil, nil)
|
||||
assertS3Error(hc.t, w, s3errors.GetAPIError(code))
|
||||
}
|
||||
|
||||
func putObjectWithHeadersBase(hc *handlerContext, bktName, objName string, headers map[string]string, box *accessbox.Box, data []byte) *httptest.ResponseRecorder {
|
||||
body := bytes.NewReader(data)
|
||||
w, r := prepareTestPayloadRequest(hc, bktName, objName, body)
|
||||
|
||||
for k, v := range headers {
|
||||
r.Header.Set(k, v)
|
||||
}
|
||||
|
||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
hc.Handler().PutObjectHandler(w, r)
|
||||
return w
|
||||
tc.Handler().PutBucketACLHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
}
|
||||
|
|
|
@ -1,20 +1,12 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -23,9 +15,7 @@ type (
|
|||
log *zap.Logger
|
||||
obj layer.Client
|
||||
notificator Notificator
|
||||
cfg Config
|
||||
ape APE
|
||||
frostfsid FrostFSID
|
||||
cfg *Config
|
||||
}
|
||||
|
||||
Notificator interface {
|
||||
|
@ -34,52 +24,41 @@ type (
|
|||
}
|
||||
|
||||
// Config contains data which handler needs to keep.
|
||||
Config interface {
|
||||
DefaultPlacementPolicy(namespace string) netmap.PlacementPolicy
|
||||
PlacementPolicy(namespace, constraint string) (netmap.PlacementPolicy, bool)
|
||||
CopiesNumbers(namespace, constraint string) ([]uint32, bool)
|
||||
DefaultCopiesNumbers(namespace string) []uint32
|
||||
NewXMLDecoder(io.Reader) *xml.Decoder
|
||||
DefaultMaxAge() int
|
||||
NotificatorEnabled() bool
|
||||
ResolveZoneList() []string
|
||||
IsResolveListAllow() bool
|
||||
BypassContentEncodingInChunks() bool
|
||||
MD5Enabled() bool
|
||||
ACLEnabled() bool
|
||||
Config struct {
|
||||
Policy PlacementPolicy
|
||||
DefaultMaxAge int
|
||||
NotificatorEnabled bool
|
||||
CopiesNumber uint32
|
||||
ResolveZoneList []string
|
||||
IsResolveListAllow bool // True if ResolveZoneList contains allowed zones
|
||||
}
|
||||
|
||||
FrostFSID interface {
|
||||
GetUserAddress(account, user string) (string, error)
|
||||
GetUserKey(account, name string) (string, error)
|
||||
PlacementPolicy interface {
|
||||
Default() netmap.PlacementPolicy
|
||||
Get(string) (netmap.PlacementPolicy, bool)
|
||||
}
|
||||
)
|
||||
|
||||
// APE is Access Policy Engine that needs to save policy and acl info to different places.
|
||||
APE interface {
|
||||
PutBucketPolicy(ns string, cnrID cid.ID, policy []byte, chains []*chain.Chain) error
|
||||
DeleteBucketPolicy(ns string, cnrID cid.ID, chainIDs []chain.ID) error
|
||||
GetBucketPolicy(ns string, cnrID cid.ID) ([]byte, error)
|
||||
SaveACLChains(cid string, chains []*chain.Chain) error
|
||||
}
|
||||
const (
|
||||
// DefaultPolicy is a default policy of placing containers in FrostFS if it's not set at the request.
|
||||
DefaultPolicy = "REP 3"
|
||||
// DefaultCopiesNumber is a default number of object copies that is enough to consider put successful if it's not set in config.
|
||||
DefaultCopiesNumber uint32 = 0
|
||||
)
|
||||
|
||||
var _ api.Handler = (*handler)(nil)
|
||||
|
||||
// New creates new api.Handler using given logger and client.
|
||||
func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg Config, storage APE, ffsid FrostFSID) (api.Handler, error) {
|
||||
func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg *Config) (api.Handler, error) {
|
||||
switch {
|
||||
case obj == nil:
|
||||
return nil, errors.New("empty FrostFS Object Layer")
|
||||
case log == nil:
|
||||
return nil, errors.New("empty logger")
|
||||
case storage == nil:
|
||||
return nil, errors.New("empty policy storage")
|
||||
case ffsid == nil:
|
||||
return nil, errors.New("empty frostfsid")
|
||||
}
|
||||
|
||||
if !cfg.NotificatorEnabled() {
|
||||
log.Warn(logs.NotificatorIsDisabledS3WontProduceNotificationEvents)
|
||||
if !cfg.NotificatorEnabled {
|
||||
log.Warn("notificator is disabled, s3 won't produce notification events")
|
||||
} else if notificator == nil {
|
||||
return nil, errors.New("empty notificator")
|
||||
}
|
||||
|
@ -88,49 +67,6 @@ func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg Config,
|
|||
log: log,
|
||||
obj: obj,
|
||||
cfg: cfg,
|
||||
ape: storage,
|
||||
notificator: notificator,
|
||||
frostfsid: ffsid,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// pickCopiesNumbers chooses the return values following this logic:
|
||||
// 1) array of copies numbers sent in request's header has the highest priority.
|
||||
// 2) array of copies numbers with corresponding location constraint provided in the config file.
|
||||
// 3) default copies number from the config file wrapped into array.
|
||||
func (h *handler) pickCopiesNumbers(metadata map[string]string, namespace, locationConstraint string) ([]uint32, error) {
|
||||
copiesNumbersStr, ok := metadata[layer.AttributeFrostfsCopiesNumber]
|
||||
if ok {
|
||||
result, err := parseCopiesNumbers(copiesNumbersStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
copiesNumbers, ok := h.cfg.CopiesNumbers(namespace, locationConstraint)
|
||||
if ok {
|
||||
return copiesNumbers, nil
|
||||
}
|
||||
|
||||
return h.cfg.DefaultCopiesNumbers(namespace), nil
|
||||
}
|
||||
|
||||
func parseCopiesNumbers(copiesNumbersStr string) ([]uint32, error) {
|
||||
var result []uint32
|
||||
copiesNumbersSplit := strings.Split(copiesNumbersStr, ",")
|
||||
|
||||
for i := range copiesNumbersSplit {
|
||||
item := strings.ReplaceAll(copiesNumbersSplit[i], " ", "")
|
||||
if len(item) == 0 {
|
||||
continue
|
||||
}
|
||||
copiesNumber, err := strconv.ParseUint(item, 10, 32)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pasrse copies number: %w", err)
|
||||
}
|
||||
result = append(result, uint32(copiesNumber))
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
|
|
@ -1,69 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCopiesNumberPicker(t *testing.T) {
|
||||
var locationConstraints = map[string][]uint32{}
|
||||
locationConstraint1 := "one"
|
||||
locationConstraint2 := "two"
|
||||
locationConstraints[locationConstraint1] = []uint32{2, 3, 4}
|
||||
|
||||
config := &configMock{
|
||||
copiesNumbers: locationConstraints,
|
||||
defaultCopiesNumbers: []uint32{1},
|
||||
}
|
||||
h := handler{
|
||||
cfg: config,
|
||||
}
|
||||
|
||||
metadata := map[string]string{}
|
||||
|
||||
t.Run("pick default copies number", func(t *testing.T) {
|
||||
metadata["somekey1"] = "5, 6, 7"
|
||||
expectedCopiesNumbers := []uint32{1}
|
||||
|
||||
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, "", locationConstraint2)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expectedCopiesNumbers, actualCopiesNumbers)
|
||||
})
|
||||
|
||||
t.Run("pick copies number vector according to location constraint", func(t *testing.T) {
|
||||
metadata["somekey2"] = "6, 7, 8"
|
||||
expectedCopiesNumbers := []uint32{2, 3, 4}
|
||||
|
||||
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, "", locationConstraint1)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expectedCopiesNumbers, actualCopiesNumbers)
|
||||
})
|
||||
|
||||
t.Run("pick copies number from metadata", func(t *testing.T) {
|
||||
metadata["frostfs-copies-number"] = "7, 8, 9"
|
||||
expectedCopiesNumbers := []uint32{7, 8, 9}
|
||||
|
||||
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, "", locationConstraint2)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expectedCopiesNumbers, actualCopiesNumbers)
|
||||
})
|
||||
|
||||
t.Run("pick copies number from metadata with no space", func(t *testing.T) {
|
||||
metadata["frostfs-copies-number"] = "7,8,9"
|
||||
expectedCopiesNumbers := []uint32{7, 8, 9}
|
||||
|
||||
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, "", locationConstraint2)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expectedCopiesNumbers, actualCopiesNumbers)
|
||||
})
|
||||
|
||||
t.Run("pick copies number from metadata with trailing comma", func(t *testing.T) {
|
||||
metadata["frostfs-copies-number"] = "11, 12, 13, "
|
||||
expectedCopiesNumbers := []uint32{11, 12, 13}
|
||||
|
||||
actualCopiesNumbers, err := h.pickCopiesNumbers(metadata, "", locationConstraint2)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expectedCopiesNumbers, actualCopiesNumbers)
|
||||
})
|
||||
}
|
|
@ -1,18 +1,15 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -20,7 +17,7 @@ type (
|
|||
GetObjectAttributesResponse struct {
|
||||
ETag string `xml:"ETag,omitempty"`
|
||||
Checksum *Checksum `xml:"Checksum,omitempty"`
|
||||
ObjectSize uint64 `xml:"ObjectSize,omitempty"`
|
||||
ObjectSize int64 `xml:"ObjectSize,omitempty"`
|
||||
StorageClass string `xml:"StorageClass,omitempty"`
|
||||
ObjectParts *ObjectParts `xml:"ObjectParts,omitempty"`
|
||||
}
|
||||
|
@ -70,7 +67,7 @@ var validAttributes = map[string]struct{}{
|
|||
}
|
||||
|
||||
func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
params, err := parseGetObjectAttributeArgs(r)
|
||||
if err != nil {
|
||||
|
@ -108,7 +105,7 @@ func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Requ
|
|||
return
|
||||
}
|
||||
|
||||
if err = checkPreconditions(info, params.Conditional, h.cfg.MD5Enabled()); err != nil {
|
||||
if err = checkPreconditions(info, params.Conditional); err != nil {
|
||||
h.logAndSendError(w, "precondition failed", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
@ -119,14 +116,14 @@ func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Requ
|
|||
return
|
||||
}
|
||||
|
||||
response, err := encodeToObjectAttributesResponse(info, params, h.cfg.MD5Enabled())
|
||||
response, err := encodeToObjectAttributesResponse(info, params)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't encode object info to response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
writeAttributesHeaders(w.Header(), extendedInfo, bktSettings.Unversioned())
|
||||
if err = middleware.EncodeToResponse(w, response); err != nil {
|
||||
if err = api.EncodeToResponse(w, response); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
@ -137,7 +134,7 @@ func writeAttributesHeaders(h http.Header, info *data.ExtendedObjectInfo, isBuck
|
|||
h.Set(api.AmzVersionID, info.Version())
|
||||
}
|
||||
|
||||
if info.NodeVersion.IsDeleteMarker {
|
||||
if info.NodeVersion.IsDeleteMarker() {
|
||||
h.Set(api.AmzDeleteMarker, strconv.FormatBool(true))
|
||||
}
|
||||
|
||||
|
@ -181,23 +178,19 @@ func parseGetObjectAttributeArgs(r *http.Request) (*GetObjectAttributesArgs, err
|
|||
return res, err
|
||||
}
|
||||
|
||||
func encodeToObjectAttributesResponse(info *data.ObjectInfo, p *GetObjectAttributesArgs, md5Enabled bool) (*GetObjectAttributesResponse, error) {
|
||||
func encodeToObjectAttributesResponse(info *data.ObjectInfo, p *GetObjectAttributesArgs) (*GetObjectAttributesResponse, error) {
|
||||
resp := &GetObjectAttributesResponse{}
|
||||
|
||||
for _, attr := range p.Attributes {
|
||||
switch attr {
|
||||
case eTag:
|
||||
resp.ETag = data.Quote(info.ETag(md5Enabled))
|
||||
resp.ETag = info.HashSum
|
||||
case storageClass:
|
||||
resp.StorageClass = api.DefaultStorageClass
|
||||
resp.StorageClass = "STANDARD"
|
||||
case objectSize:
|
||||
resp.ObjectSize = info.Size
|
||||
case checksum:
|
||||
checksumBytes, err := hex.DecodeString(info.HashSum)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("form upload attributes: %w", err)
|
||||
}
|
||||
resp.Checksum = &Checksum{ChecksumSHA256: base64.StdEncoding.EncodeToString(checksumBytes)}
|
||||
resp.Checksum = &Checksum{ChecksumSHA256: info.HashSum}
|
||||
case objectParts:
|
||||
parts, err := formUploadAttributes(info, p.MaxParts, p.PartNumberMarker)
|
||||
if err != nil {
|
||||
|
@ -225,15 +218,10 @@ func formUploadAttributes(info *data.ObjectInfo, maxParts, marker int) (*ObjectP
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid completed part: %w", err)
|
||||
}
|
||||
// ETag value contains SHA256 checksum.
|
||||
checksumBytes, err := hex.DecodeString(part.ETag)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid sha256 checksum in completed part: %w", err)
|
||||
}
|
||||
parts[i] = Part{
|
||||
PartNumber: part.PartNumber,
|
||||
Size: int(part.Size),
|
||||
ChecksumSHA256: base64.StdEncoding.EncodeToString(checksumBytes),
|
||||
ChecksumSHA256: part.ETag,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -1,12 +1,10 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -19,20 +17,18 @@ func TestGetObjectPartsAttributes(t *testing.T) {
|
|||
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
putObject(hc, bktName, objName)
|
||||
putObject(t, hc, bktName, objName)
|
||||
result := getObjectAttributes(hc, bktName, objName, objectParts)
|
||||
require.Nil(t, result.ObjectParts)
|
||||
|
||||
multipartUpload := createMultipartUpload(hc, bktName, objMultipartName, map[string]string{})
|
||||
etag, _ := uploadPart(hc, bktName, objMultipartName, multipartUpload.UploadID, 1, partSize)
|
||||
completeMultipartUpload(hc, bktName, objMultipartName, multipartUpload.UploadID, []string{etag})
|
||||
etagBytes, err := hex.DecodeString(etag[1 : len(etag)-1])
|
||||
require.NoError(t, err)
|
||||
|
||||
result = getObjectAttributes(hc, bktName, objMultipartName, objectParts)
|
||||
require.NotNil(t, result.ObjectParts)
|
||||
require.Len(t, result.ObjectParts.Parts, 1)
|
||||
require.Equal(t, base64.StdEncoding.EncodeToString(etagBytes), result.ObjectParts.Parts[0].ChecksumSHA256)
|
||||
require.Equal(t, etag, result.ObjectParts.Parts[0].ChecksumSHA256)
|
||||
require.Equal(t, partSize, result.ObjectParts.Parts[0].Size)
|
||||
require.Equal(t, 1, result.ObjectParts.PartsCount)
|
||||
}
|
||||
|
|
|
@ -6,14 +6,12 @@ import (
|
|||
"regexp"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -48,10 +46,9 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
tagSet map[string]string
|
||||
sessionTokenEACL *session.Container
|
||||
|
||||
ctx = r.Context()
|
||||
reqInfo = middleware.GetReqInfo(ctx)
|
||||
reqInfo = api.GetReqInfo(r.Context())
|
||||
|
||||
cannedACLStatus = aclHeadersStatus(r)
|
||||
containsACL = containsACLHeaders(r)
|
||||
)
|
||||
|
||||
src := r.Header.Get(api.AmzCopySource)
|
||||
|
@ -87,65 +84,26 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
settings, err := h.obj.GetBucketSettings(ctx, dstBktInfo)
|
||||
settings, err := h.obj.GetBucketSettings(r.Context(), dstBktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
apeEnabled := dstBktInfo.APEEnabled || settings.CannedACL != ""
|
||||
if apeEnabled && cannedACLStatus == aclStatusYes {
|
||||
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
||||
return
|
||||
}
|
||||
|
||||
needUpdateEACLTable := !(apeEnabled || cannedACLStatus == aclStatusNo)
|
||||
if needUpdateEACLTable {
|
||||
if sessionTokenEACL, err = getSessionTokenSetEACL(ctx); err != nil {
|
||||
if containsACL {
|
||||
if sessionTokenEACL, err = getSessionTokenSetEACL(r.Context()); err != nil {
|
||||
h.logAndSendError(w, "could not get eacl session token from a box", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
extendedSrcObjInfo, err := h.obj.GetExtendedObjectInfo(ctx, srcObjPrm)
|
||||
extendedSrcObjInfo, err := h.obj.GetExtendedObjectInfo(r.Context(), srcObjPrm)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not find object", reqInfo, err)
|
||||
return
|
||||
}
|
||||
srcObjInfo := extendedSrcObjInfo.ObjectInfo
|
||||
|
||||
srcEncryptionParams, err := formCopySourceEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
}
|
||||
dstEncryptionParams, err := formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = srcEncryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(srcObjInfo.Headers)); err != nil {
|
||||
if errors.IsS3Error(err, errors.ErrInvalidEncryptionParameters) || errors.IsS3Error(err, errors.ErrSSEEncryptedObject) ||
|
||||
errors.IsS3Error(err, errors.ErrInvalidSSECustomerParameters) {
|
||||
h.logAndSendError(w, "encryption doesn't match object", reqInfo, err, zap.Error(err))
|
||||
return
|
||||
}
|
||||
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
var dstSize uint64
|
||||
srcSize, err := layer.GetObjectSize(srcObjInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "failed to get source object size", reqInfo, err)
|
||||
return
|
||||
} else if srcSize > layer.UploadMaxSize { // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
|
||||
h.logAndSendError(w, "too bid object to copy with single copy operation, use multipart upload copy instead", reqInfo, errors.GetAPIError(errors.ErrInvalidRequestLargeCopy))
|
||||
return
|
||||
}
|
||||
dstSize = srcSize
|
||||
|
||||
args, err := parseCopyObjectArgs(r.Header)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not parse request params", reqInfo, err)
|
||||
|
@ -168,8 +126,8 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
} else {
|
||||
tagPrm := &data.GetObjectTaggingParams{
|
||||
ObjectVersion: &data.ObjectVersion{
|
||||
tagPrm := &layer.GetObjectTaggingParams{
|
||||
ObjectVersion: &layer.ObjectVersion{
|
||||
BktInfo: srcObjPrm.BktInfo,
|
||||
ObjectName: srcObject,
|
||||
VersionID: srcObjInfo.VersionID(),
|
||||
|
@ -177,14 +135,25 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
NodeVersion: extendedSrcObjInfo.NodeVersion,
|
||||
}
|
||||
|
||||
_, tagSet, err = h.obj.GetObjectTagging(ctx, tagPrm)
|
||||
_, tagSet, err = h.obj.GetObjectTagging(r.Context(), tagPrm)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get object tagging", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err = checkPreconditions(srcObjInfo, args.Conditional, h.cfg.MD5Enabled()); err != nil {
|
||||
encryptionParams, err := formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = encryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(srcObjInfo.Headers)); err != nil {
|
||||
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
if err = checkPreconditions(srcObjInfo, args.Conditional); err != nil {
|
||||
h.logAndSendError(w, "precondition failed", reqInfo, errors.GetAPIError(errors.ErrPreconditionFailed))
|
||||
return
|
||||
}
|
||||
|
@ -193,53 +162,48 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
if len(srcObjInfo.ContentType) > 0 {
|
||||
srcObjInfo.Headers[api.ContentType] = srcObjInfo.ContentType
|
||||
}
|
||||
metadata = makeCopyMap(srcObjInfo.Headers)
|
||||
filterMetadataMap(metadata)
|
||||
metadata = srcObjInfo.Headers
|
||||
} else if contentType := r.Header.Get(api.ContentType); len(contentType) > 0 {
|
||||
metadata[api.ContentType] = contentType
|
||||
}
|
||||
|
||||
params := &layer.CopyObjectParams{
|
||||
SrcVersioned: srcObjPrm.Versioned(),
|
||||
SrcObject: srcObjInfo,
|
||||
ScrBktInfo: srcObjPrm.BktInfo,
|
||||
DstBktInfo: dstBktInfo,
|
||||
DstObject: reqInfo.ObjectName,
|
||||
DstSize: dstSize,
|
||||
Header: metadata,
|
||||
SrcEncryption: srcEncryptionParams,
|
||||
DstEncryption: dstEncryptionParams,
|
||||
}
|
||||
|
||||
params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, reqInfo.Namespace, dstBktInfo.LocationConstraint)
|
||||
copiesNumber, err := getCopiesNumberOrDefault(metadata, h.cfg.CopiesNumber)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
params.Lock, err = formObjectLock(ctx, dstBktInfo, settings.LockConfiguration, r.Header)
|
||||
params := &layer.CopyObjectParams{
|
||||
SrcObject: srcObjInfo,
|
||||
ScrBktInfo: srcObjPrm.BktInfo,
|
||||
DstBktInfo: dstBktInfo,
|
||||
DstObject: reqInfo.ObjectName,
|
||||
SrcSize: srcObjInfo.Size,
|
||||
Header: metadata,
|
||||
Encryption: encryptionParams,
|
||||
CopiesNuber: copiesNumber,
|
||||
}
|
||||
|
||||
params.Lock, err = formObjectLock(r.Context(), dstBktInfo, settings.LockConfiguration, r.Header)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not form object lock", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
additional := []zap.Field{zap.String("src_bucket_name", srcBucket), zap.String("src_object_name", srcObject)}
|
||||
extendedDstObjInfo, err := h.obj.CopyObject(ctx, params)
|
||||
extendedDstObjInfo, err := h.obj.CopyObject(r.Context(), params)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't copy object", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
dstObjInfo := extendedDstObjInfo.ObjectInfo
|
||||
|
||||
if err = middleware.EncodeToResponse(w, &CopyObjectResponse{
|
||||
LastModified: dstObjInfo.Created.UTC().Format(time.RFC3339),
|
||||
ETag: data.Quote(dstObjInfo.ETag(h.cfg.MD5Enabled())),
|
||||
}); err != nil {
|
||||
if err = api.EncodeToResponse(w, &CopyObjectResponse{LastModified: dstObjInfo.Created.UTC().Format(time.RFC3339), ETag: dstObjInfo.HashSum}); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
|
||||
if needUpdateEACLTable {
|
||||
if containsACL {
|
||||
newEaclTable, err := h.getNewEAclTable(r, dstBktInfo, dstObjInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get new eacl table", reqInfo, err)
|
||||
|
@ -252,15 +216,15 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
SessionToken: sessionTokenEACL,
|
||||
}
|
||||
|
||||
if err = h.obj.PutBucketACL(ctx, p); err != nil {
|
||||
if err = h.obj.PutBucketACL(r.Context(), p); err != nil {
|
||||
h.logAndSendError(w, "could not put bucket acl", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if tagSet != nil {
|
||||
tagPrm := &data.PutObjectTaggingParams{
|
||||
ObjectVersion: &data.ObjectVersion{
|
||||
tagPrm := &layer.PutObjectTaggingParams{
|
||||
ObjectVersion: &layer.ObjectVersion{
|
||||
BktInfo: dstBktInfo,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
VersionID: dstObjInfo.VersionID(),
|
||||
|
@ -268,45 +232,33 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
TagSet: tagSet,
|
||||
NodeVersion: extendedDstObjInfo.NodeVersion,
|
||||
}
|
||||
if _, err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
|
||||
if _, err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
|
||||
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID))
|
||||
h.log.Info("object is copied",
|
||||
zap.String("bucket", dstObjInfo.Bucket),
|
||||
zap.String("object", dstObjInfo.Name),
|
||||
zap.Stringer("object_id", dstObjInfo.ID))
|
||||
|
||||
s := &SendNotificationParams{
|
||||
Event: EventObjectCreatedCopy,
|
||||
NotificationInfo: data.NotificationInfoFromObject(dstObjInfo, h.cfg.MD5Enabled()),
|
||||
NotificationInfo: data.NotificationInfoFromObject(dstObjInfo),
|
||||
BktInfo: dstBktInfo,
|
||||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
if err = h.sendNotifications(r.Context(), s); err != nil {
|
||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||
}
|
||||
|
||||
if dstEncryptionParams.Enabled() {
|
||||
if encryptionParams.Enabled() {
|
||||
addSSECHeaders(w.Header(), r.Header)
|
||||
}
|
||||
}
|
||||
|
||||
func makeCopyMap(headers map[string]string) map[string]string {
|
||||
res := make(map[string]string, len(headers))
|
||||
for key, val := range headers {
|
||||
res[key] = val
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func filterMetadataMap(metadata map[string]string) {
|
||||
delete(metadata, layer.MultipartObjectSize) // object payload will be real one rather than list of compound parts
|
||||
for key := range layer.EncryptionMetadata {
|
||||
delete(metadata, key)
|
||||
}
|
||||
}
|
||||
|
||||
func isCopyingToItselfForbidden(reqInfo *middleware.ReqInfo, srcBucket string, srcObject string, settings *data.BucketSettings, args *copyObjectArgs) bool {
|
||||
func isCopyingToItselfForbidden(reqInfo *api.ReqInfo, srcBucket string, srcObject string, settings *data.BucketSettings, args *copyObjectArgs) bool {
|
||||
if reqInfo.BucketName != srcBucket || reqInfo.ObjectName != srcObject {
|
||||
return false
|
||||
}
|
||||
|
@ -321,8 +273,8 @@ func isCopyingToItselfForbidden(reqInfo *middleware.ReqInfo, srcBucket string, s
|
|||
func parseCopyObjectArgs(headers http.Header) (*copyObjectArgs, error) {
|
||||
var err error
|
||||
args := &conditionalArgs{
|
||||
IfMatch: data.UnQuote(headers.Get(api.AmzCopyIfMatch)),
|
||||
IfNoneMatch: data.UnQuote(headers.Get(api.AmzCopyIfNoneMatch)),
|
||||
IfMatch: headers.Get(api.AmzCopyIfMatch),
|
||||
IfNoneMatch: headers.Get(api.AmzCopyIfNoneMatch),
|
||||
}
|
||||
|
||||
if args.IfModifiedSince, err = parseHTTPTime(headers.Get(api.AmzCopyIfModifiedSince)); err != nil {
|
||||
|
|
|
@ -1,21 +1,12 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -24,7 +15,6 @@ type CopyMeta struct {
|
|||
Tags map[string]string
|
||||
MetadataDirective string
|
||||
Metadata map[string]string
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
func TestCopyWithTaggingDirective(t *testing.T) {
|
||||
|
@ -39,14 +29,14 @@ func TestCopyWithTaggingDirective(t *testing.T) {
|
|||
copyMeta := CopyMeta{
|
||||
Tags: map[string]string{"key2": "val"},
|
||||
}
|
||||
copyObject(tc, bktName, objName, objToCopy, copyMeta, http.StatusOK)
|
||||
copyObject(t, tc, bktName, objName, objToCopy, copyMeta, http.StatusOK)
|
||||
tagging := getObjectTagging(t, tc, bktName, objToCopy, emptyVersion)
|
||||
require.Len(t, tagging.TagSet, 1)
|
||||
require.Equal(t, "key", tagging.TagSet[0].Key)
|
||||
require.Equal(t, "val", tagging.TagSet[0].Value)
|
||||
|
||||
copyMeta.TaggingDirective = replaceDirective
|
||||
copyObject(tc, bktName, objName, objToCopy2, copyMeta, http.StatusOK)
|
||||
copyObject(t, tc, bktName, objName, objToCopy2, copyMeta, http.StatusOK)
|
||||
tagging = getObjectTagging(t, tc, bktName, objToCopy2, emptyVersion)
|
||||
require.Len(t, tagging.TagSet, 1)
|
||||
require.Equal(t, "key2", tagging.TagSet[0].Key)
|
||||
|
@ -61,213 +51,20 @@ func TestCopyToItself(t *testing.T) {
|
|||
|
||||
copyMeta := CopyMeta{MetadataDirective: replaceDirective}
|
||||
|
||||
copyObject(tc, bktName, objName, objName, CopyMeta{}, http.StatusBadRequest)
|
||||
copyObject(tc, bktName, objName, objName, copyMeta, http.StatusOK)
|
||||
copyObject(t, tc, bktName, objName, objName, CopyMeta{}, http.StatusBadRequest)
|
||||
copyObject(t, tc, bktName, objName, objName, copyMeta, http.StatusOK)
|
||||
|
||||
putBucketVersioning(t, tc, bktName, true)
|
||||
copyObject(tc, bktName, objName, objName, CopyMeta{}, http.StatusOK)
|
||||
copyObject(tc, bktName, objName, objName, copyMeta, http.StatusOK)
|
||||
copyObject(t, tc, bktName, objName, objName, CopyMeta{}, http.StatusOK)
|
||||
copyObject(t, tc, bktName, objName, objName, copyMeta, http.StatusOK)
|
||||
|
||||
putBucketVersioning(t, tc, bktName, false)
|
||||
copyObject(tc, bktName, objName, objName, CopyMeta{}, http.StatusOK)
|
||||
copyObject(tc, bktName, objName, objName, copyMeta, http.StatusOK)
|
||||
copyObject(t, tc, bktName, objName, objName, CopyMeta{}, http.StatusOK)
|
||||
copyObject(t, tc, bktName, objName, objName, copyMeta, http.StatusOK)
|
||||
}
|
||||
|
||||
func TestCopyMultipart(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-copy", "object-for-copy"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
partSize := layer.UploadMinSize
|
||||
objLen := 6 * partSize
|
||||
headers := map[string]string{}
|
||||
|
||||
data := multipartUpload(hc, bktName, objName, headers, objLen, partSize)
|
||||
require.Equal(t, objLen, len(data))
|
||||
|
||||
objToCopy := "copy-target"
|
||||
var copyMeta CopyMeta
|
||||
copyObject(hc, bktName, objName, objToCopy, copyMeta, http.StatusOK)
|
||||
|
||||
copiedData, _ := getObject(hc, bktName, objToCopy)
|
||||
equalDataSlices(t, data, copiedData)
|
||||
|
||||
result := getObjectAttributes(hc, bktName, objToCopy, objectParts)
|
||||
require.NotNil(t, result.ObjectParts)
|
||||
|
||||
objToCopy2 := "copy-target2"
|
||||
copyMeta.MetadataDirective = replaceDirective
|
||||
copyObject(hc, bktName, objName, objToCopy2, copyMeta, http.StatusOK)
|
||||
|
||||
result = getObjectAttributes(hc, bktName, objToCopy2, objectParts)
|
||||
require.Nil(t, result.ObjectParts)
|
||||
|
||||
copiedData, _ = getObject(hc, bktName, objToCopy2)
|
||||
equalDataSlices(t, data, copiedData)
|
||||
}
|
||||
|
||||
func TestCopyEncryptedToUnencrypted(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
bktName, srcObjName := "bucket-for-copy", "object-for-copy"
|
||||
key1 := []byte("firstencriptionkeyofsourceobject")
|
||||
key1Md5 := md5.Sum(key1)
|
||||
key2 := []byte("anotherencriptionkeysourceobject")
|
||||
key2Md5 := md5.Sum(key2)
|
||||
bktInfo := createTestBucket(tc, bktName)
|
||||
|
||||
srcEnc, err := encryption.NewParams(key1)
|
||||
require.NoError(t, err)
|
||||
srcObjInfo := createTestObject(tc, bktInfo, srcObjName, *srcEnc)
|
||||
require.True(t, containEncryptionMetadataHeaders(srcObjInfo.Headers))
|
||||
|
||||
dstObjName := "copy-object"
|
||||
|
||||
// empty copy-source-sse headers
|
||||
w, r := prepareTestRequest(tc, bktName, dstObjName, nil)
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
r.Header.Set(api.AmzCopySource, bktName+"/"+srcObjName)
|
||||
tc.Handler().CopyObjectHandler(w, r)
|
||||
|
||||
assertStatus(t, w, http.StatusBadRequest)
|
||||
assertS3Error(t, w, errors.GetAPIError(errors.ErrSSEEncryptedObject))
|
||||
|
||||
// empty copy-source-sse-custom-key
|
||||
w, r = prepareTestRequest(tc, bktName, dstObjName, nil)
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
r.Header.Set(api.AmzCopySource, bktName+"/"+srcObjName)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
|
||||
tc.Handler().CopyObjectHandler(w, r)
|
||||
|
||||
assertStatus(t, w, http.StatusBadRequest)
|
||||
assertS3Error(t, w, errors.GetAPIError(errors.ErrMissingSSECustomerKey))
|
||||
|
||||
// empty copy-source-sse-custom-algorithm
|
||||
w, r = prepareTestRequest(tc, bktName, dstObjName, nil)
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
r.Header.Set(api.AmzCopySource, bktName+"/"+srcObjName)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKey, base64.StdEncoding.EncodeToString(key1))
|
||||
tc.Handler().CopyObjectHandler(w, r)
|
||||
|
||||
assertStatus(t, w, http.StatusBadRequest)
|
||||
assertS3Error(t, w, errors.GetAPIError(errors.ErrMissingSSECustomerAlgorithm))
|
||||
|
||||
// invalid copy-source-sse-custom-key
|
||||
w, r = prepareTestRequest(tc, bktName, dstObjName, nil)
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
r.Header.Set(api.AmzCopySource, bktName+"/"+srcObjName)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKey, base64.StdEncoding.EncodeToString(key2))
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKeyMD5, base64.StdEncoding.EncodeToString(key2Md5[:]))
|
||||
tc.Handler().CopyObjectHandler(w, r)
|
||||
|
||||
assertStatus(t, w, http.StatusBadRequest)
|
||||
assertS3Error(t, w, errors.GetAPIError(errors.ErrInvalidSSECustomerParameters))
|
||||
|
||||
// success copy
|
||||
w, r = prepareTestRequest(tc, bktName, dstObjName, nil)
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
r.Header.Set(api.AmzCopySource, bktName+"/"+srcObjName)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKey, base64.StdEncoding.EncodeToString(key1))
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKeyMD5, base64.StdEncoding.EncodeToString(key1Md5[:]))
|
||||
tc.Handler().CopyObjectHandler(w, r)
|
||||
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
dstObjInfo, err := tc.Layer().GetObjectInfo(tc.Context(), &layer.HeadObjectParams{BktInfo: bktInfo, Object: dstObjName})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, srcObjInfo.Headers[layer.AttributeDecryptedSize], strconv.Itoa(int(dstObjInfo.Size)))
|
||||
require.False(t, containEncryptionMetadataHeaders(dstObjInfo.Headers))
|
||||
}
|
||||
|
||||
func TestCopyUnencryptedToEncrypted(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
bktName, srcObjName := "bucket-for-copy", "object-for-copy"
|
||||
key := []byte("firstencriptionkeyofsourceobject")
|
||||
keyMd5 := md5.Sum(key)
|
||||
bktInfo := createTestBucket(tc, bktName)
|
||||
|
||||
srcObjInfo := createTestObject(tc, bktInfo, srcObjName, encryption.Params{})
|
||||
require.False(t, containEncryptionMetadataHeaders(srcObjInfo.Headers))
|
||||
|
||||
dstObjName := "copy-object"
|
||||
|
||||
// invalid copy-source-sse headers
|
||||
w, r := prepareTestRequest(tc, bktName, dstObjName, nil)
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
r.Header.Set(api.AmzCopySource, bktName+"/"+srcObjName)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKey, base64.StdEncoding.EncodeToString(key))
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMd5[:]))
|
||||
tc.Handler().CopyObjectHandler(w, r)
|
||||
|
||||
assertStatus(t, w, http.StatusBadRequest)
|
||||
assertS3Error(t, w, errors.GetAPIError(errors.ErrInvalidEncryptionParameters))
|
||||
|
||||
// success copy
|
||||
w, r = prepareTestRequest(tc, bktName, dstObjName, nil)
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
r.Header.Set(api.AmzCopySource, bktName+"/"+srcObjName)
|
||||
r.Header.Set(api.AmzServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
|
||||
r.Header.Set(api.AmzServerSideEncryptionCustomerKey, base64.StdEncoding.EncodeToString(key))
|
||||
r.Header.Set(api.AmzServerSideEncryptionCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMd5[:]))
|
||||
tc.Handler().CopyObjectHandler(w, r)
|
||||
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
dstObjInfo, err := tc.Layer().GetObjectInfo(tc.Context(), &layer.HeadObjectParams{BktInfo: bktInfo, Object: dstObjName})
|
||||
require.NoError(t, err)
|
||||
require.True(t, containEncryptionMetadataHeaders(dstObjInfo.Headers))
|
||||
require.Equal(t, strconv.Itoa(int(srcObjInfo.Size)), dstObjInfo.Headers[layer.AttributeDecryptedSize])
|
||||
}
|
||||
|
||||
func TestCopyEncryptedToEncryptedWithAnotherKey(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
bktName, srcObjName := "bucket-for-copy", "object-for-copy"
|
||||
key1 := []byte("firstencriptionkeyofsourceobject")
|
||||
key1Md5 := md5.Sum(key1)
|
||||
key2 := []byte("anotherencriptionkeysourceobject")
|
||||
key2Md5 := md5.Sum(key2)
|
||||
bktInfo := createTestBucket(tc, bktName)
|
||||
|
||||
srcEnc, err := encryption.NewParams(key1)
|
||||
require.NoError(t, err)
|
||||
srcObjInfo := createTestObject(tc, bktInfo, srcObjName, *srcEnc)
|
||||
require.True(t, containEncryptionMetadataHeaders(srcObjInfo.Headers))
|
||||
|
||||
dstObjName := "copy-object"
|
||||
|
||||
w, r := prepareTestRequest(tc, bktName, dstObjName, nil)
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
r.Header.Set(api.AmzCopySource, bktName+"/"+srcObjName)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKey, base64.StdEncoding.EncodeToString(key1))
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKeyMD5, base64.StdEncoding.EncodeToString(key1Md5[:]))
|
||||
r.Header.Set(api.AmzServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
|
||||
r.Header.Set(api.AmzServerSideEncryptionCustomerKey, base64.StdEncoding.EncodeToString(key2))
|
||||
r.Header.Set(api.AmzServerSideEncryptionCustomerKeyMD5, base64.StdEncoding.EncodeToString(key2Md5[:]))
|
||||
tc.Handler().CopyObjectHandler(w, r)
|
||||
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
dstObjInfo, err := tc.Layer().GetObjectInfo(tc.Context(), &layer.HeadObjectParams{BktInfo: bktInfo, Object: dstObjName})
|
||||
require.NoError(t, err)
|
||||
require.True(t, containEncryptionMetadataHeaders(dstObjInfo.Headers))
|
||||
require.Equal(t, srcObjInfo.Headers[layer.AttributeDecryptedSize], dstObjInfo.Headers[layer.AttributeDecryptedSize])
|
||||
}
|
||||
|
||||
func containEncryptionMetadataHeaders(headers map[string]string) bool {
|
||||
for k := range headers {
|
||||
if _, ok := layer.EncryptionMetadata[k]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func copyObject(hc *handlerContext, bktName, fromObject, toObject string, copyMeta CopyMeta, statusCode int) {
|
||||
w, r := prepareTestRequest(hc, bktName, toObject, nil)
|
||||
func copyObject(t *testing.T, tc *handlerContext, bktName, fromObject, toObject string, copyMeta CopyMeta, statusCode int) {
|
||||
w, r := prepareTestRequest(tc, bktName, toObject, nil)
|
||||
r.Header.Set(api.AmzCopySource, bktName+"/"+fromObject)
|
||||
|
||||
r.Header.Set(api.AmzMetadataDirective, copyMeta.MetadataDirective)
|
||||
|
@ -282,33 +79,28 @@ func copyObject(hc *handlerContext, bktName, fromObject, toObject string, copyMe
|
|||
}
|
||||
r.Header.Set(api.AmzTagging, tagsQuery.Encode())
|
||||
|
||||
for key, val := range copyMeta.Headers {
|
||||
r.Header.Set(key, val)
|
||||
}
|
||||
|
||||
hc.Handler().CopyObjectHandler(w, r)
|
||||
assertStatus(hc.t, w, statusCode)
|
||||
tc.Handler().CopyObjectHandler(w, r)
|
||||
assertStatus(t, w, statusCode)
|
||||
}
|
||||
|
||||
func putObjectTagging(t *testing.T, tc *handlerContext, bktName, objName string, tags map[string]string) {
|
||||
body := &data.Tagging{
|
||||
TagSet: make([]data.Tag, 0, len(tags)),
|
||||
body := &Tagging{
|
||||
TagSet: make([]Tag, 0, len(tags)),
|
||||
}
|
||||
|
||||
for key, val := range tags {
|
||||
body.TagSet = append(body.TagSet, data.Tag{
|
||||
body.TagSet = append(body.TagSet, Tag{
|
||||
Key: key,
|
||||
Value: val,
|
||||
})
|
||||
}
|
||||
|
||||
w, r := prepareTestRequest(tc, bktName, objName, body)
|
||||
middleware.GetReqInfo(r.Context()).Tagging = body
|
||||
tc.Handler().PutObjectTaggingHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, version string) *data.Tagging {
|
||||
func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, version string) *Tagging {
|
||||
query := make(url.Values)
|
||||
query.Add(api.QueryVersionID, version)
|
||||
|
||||
|
@ -316,7 +108,7 @@ func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, versio
|
|||
tc.Handler().GetObjectTaggingHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
tagging := &data.Tagging{}
|
||||
tagging := &Tagging{}
|
||||
err := xml.NewDecoder(w.Result().Body).Decode(tagging)
|
||||
require.NoError(t, err)
|
||||
return tagging
|
||||
|
|
|
@ -5,11 +5,9 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -20,7 +18,7 @@ const (
|
|||
)
|
||||
|
||||
func (h *handler) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -34,14 +32,14 @@ func (h *handler) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, cors); err != nil {
|
||||
if err = api.EncodeToResponse(w, cors); err != nil {
|
||||
h.logAndSendError(w, "could not encode cors to response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -50,15 +48,9 @@ func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
p := &layer.PutCORSParams{
|
||||
BktInfo: bktInfo,
|
||||
Reader: r.Body,
|
||||
NewDecoder: h.cfg.NewXMLDecoder,
|
||||
}
|
||||
|
||||
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||
return
|
||||
BktInfo: bktInfo,
|
||||
Reader: r.Body,
|
||||
CopiesNumber: h.cfg.CopiesNumber,
|
||||
}
|
||||
|
||||
if err = h.obj.PutBucketCORS(r.Context(), p); err != nil {
|
||||
|
@ -66,14 +58,11 @@ func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
|
||||
h.logAndSendError(w, "write response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
api.WriteSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
func (h *handler) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -96,21 +85,19 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
|
|||
if origin == "" {
|
||||
return
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
if reqInfo.BucketName == "" {
|
||||
return
|
||||
}
|
||||
bktInfo, err := h.obj.GetBucketInfo(ctx, reqInfo.BucketName)
|
||||
bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err))
|
||||
h.log.Warn("get bucket info", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
cors, err := h.obj.GetBucketCORS(ctx, bktInfo)
|
||||
cors, err := h.obj.GetBucketCORS(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err))
|
||||
h.log.Warn("get bucket cors", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -149,7 +136,7 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
|
@ -198,15 +185,12 @@ func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
|
|||
if rule.MaxAgeSeconds > 0 || rule.MaxAgeSeconds == -1 {
|
||||
w.Header().Set(api.AccessControlMaxAge, strconv.Itoa(rule.MaxAgeSeconds))
|
||||
} else {
|
||||
w.Header().Set(api.AccessControlMaxAge, strconv.Itoa(h.cfg.DefaultMaxAge()))
|
||||
w.Header().Set(api.AccessControlMaxAge, strconv.Itoa(h.cfg.DefaultMaxAge))
|
||||
}
|
||||
if o != wildcard {
|
||||
w.Header().Set(api.AccessControlAllowCredentials, "true")
|
||||
}
|
||||
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
|
||||
h.logAndSendError(w, "write response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
api.WriteSuccessResponseHeadersOnly(w)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,41 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
)
|
||||
|
||||
func TestCORSOriginWildcard(t *testing.T) {
|
||||
body := `
|
||||
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<CORSRule>
|
||||
<AllowedMethod>GET</AllowedMethod>
|
||||
<AllowedOrigin>*</AllowedOrigin>
|
||||
</CORSRule>
|
||||
</CORSConfiguration>
|
||||
`
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-cors"
|
||||
box, _ := createAccessBox(t)
|
||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
||||
r = r.WithContext(ctx)
|
||||
r.Header.Add(api.AmzACL, "public-read")
|
||||
hc.Handler().CreateBucketHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
|
||||
ctx = middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().PutBucketCorsHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
|
||||
hc.Handler().GetBucketCorsHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
}
|
|
@ -2,32 +2,25 @@ package handler
|
|||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
apistatus "github.com/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
)
|
||||
|
||||
// limitation of AWS https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
|
||||
const maxObjectsToDelete = 1000
|
||||
|
||||
// DeleteObjectsRequest -- xml carrying the object key names which should be deleted.
|
||||
type DeleteObjectsRequest struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Delete" json:"-"`
|
||||
// Element to enable quiet mode for the request
|
||||
Quiet bool `xml:"Quiet,omitempty"`
|
||||
Quiet bool
|
||||
// List of objects to be deleted
|
||||
Objects []ObjectIdentifier `xml:"Object"`
|
||||
}
|
||||
|
@ -47,10 +40,10 @@ type DeletedObject struct {
|
|||
|
||||
// DeleteError structure.
|
||||
type DeleteError struct {
|
||||
Code string `xml:"Code,omitempty"`
|
||||
Message string `xml:"Message,omitempty"`
|
||||
Key string `xml:"Key,omitempty"`
|
||||
VersionID string `xml:"VersionId,omitempty"`
|
||||
Code string
|
||||
Message string
|
||||
Key string
|
||||
VersionID string `xml:"versionId,omitempty"`
|
||||
}
|
||||
|
||||
// DeleteObjectsResponse container for multiple object deletes.
|
||||
|
@ -65,8 +58,7 @@ type DeleteObjectsResponse struct {
|
|||
}
|
||||
|
||||
func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
versionID := reqInfo.URL.Query().Get(api.QueryVersionID)
|
||||
versionedObject := []*layer.VersionedObject{{
|
||||
Name: reqInfo.ObjectName,
|
||||
|
@ -79,7 +71,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
bktSettings, err := h.obj.GetBucketSettings(ctx, bktInfo)
|
||||
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||
return
|
||||
|
@ -90,7 +82,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
Objects: versionedObject,
|
||||
Settings: bktSettings,
|
||||
}
|
||||
deletedObjects := h.obj.DeleteObjects(ctx, p)
|
||||
deletedObjects := h.obj.DeleteObjects(r.Context(), p)
|
||||
deletedObject := deletedObjects[0]
|
||||
if deletedObject.Error != nil {
|
||||
if isErrObjectLocked(deletedObject.Error) {
|
||||
|
@ -117,7 +109,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
var objID oid.ID
|
||||
if len(versionID) != 0 {
|
||||
if err = objID.DecodeString(versionID); err != nil {
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -132,8 +124,8 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
}
|
||||
|
||||
if err = h.sendNotifications(ctx, m); err != nil {
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
if err = h.sendNotifications(r.Context(), m); err != nil {
|
||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||
}
|
||||
|
||||
if deletedObject.VersionID != "" {
|
||||
|
@ -153,15 +145,15 @@ func isErrObjectLocked(err error) bool {
|
|||
switch err.(type) {
|
||||
default:
|
||||
return strings.Contains(err.Error(), "object is locked")
|
||||
case *apistatus.ObjectLocked:
|
||||
case apistatus.ObjectLocked,
|
||||
*apistatus.ObjectLocked:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteMultipleObjectsHandler handles multiple delete requests.
|
||||
func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
// Content-Md5 is required and should be set
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
|
@ -179,13 +171,8 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
|||
|
||||
// Unmarshal list of keys to be deleted.
|
||||
requested := &DeleteObjectsRequest{}
|
||||
if err := h.cfg.NewXMLDecoder(r.Body).Decode(requested); err != nil {
|
||||
h.logAndSendError(w, "couldn't decode body", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
if len(requested.Objects) == 0 || len(requested.Objects) > maxObjectsToDelete {
|
||||
h.logAndSendError(w, "number of objects to delete must be greater than 0 and less or equal to 1000", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
|
||||
if err := xml.NewDecoder(r.Body).Decode(requested); err != nil {
|
||||
h.logAndSendError(w, "couldn't decode body", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -211,20 +198,27 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
|||
return
|
||||
}
|
||||
|
||||
bktSettings, err := h.obj.GetBucketSettings(ctx, bktInfo)
|
||||
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
p := &layer.DeleteObjectParams{
|
||||
BktInfo: bktInfo,
|
||||
Objects: toRemove,
|
||||
Settings: bktSettings,
|
||||
IsMultiple: true,
|
||||
}
|
||||
deletedObjects := h.obj.DeleteObjects(ctx, p)
|
||||
marshaler := zapcore.ArrayMarshalerFunc(func(encoder zapcore.ArrayEncoder) error {
|
||||
for _, obj := range toRemove {
|
||||
encoder.AppendString(obj.String())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
p := &layer.DeleteObjectParams{
|
||||
BktInfo: bktInfo,
|
||||
Objects: toRemove,
|
||||
Settings: bktSettings,
|
||||
}
|
||||
deletedObjects := h.obj.DeleteObjects(r.Context(), p)
|
||||
|
||||
var errs []error
|
||||
for _, obj := range deletedObjects {
|
||||
if obj.Error != nil {
|
||||
code := "BadRequest"
|
||||
|
@ -237,6 +231,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
|||
Key: obj.Name,
|
||||
VersionID: obj.VersionID,
|
||||
})
|
||||
errs = append(errs, obj.Error)
|
||||
} else if !requested.Quiet {
|
||||
deletedObj := DeletedObject{
|
||||
ObjectIdentifier: ObjectIdentifier{
|
||||
|
@ -251,15 +246,22 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
|||
response.DeletedObjects = append(response.DeletedObjects, deletedObj)
|
||||
}
|
||||
}
|
||||
if len(errs) != 0 {
|
||||
fields := []zap.Field{
|
||||
zap.Array("objects", marshaler),
|
||||
zap.Errors("errors", errs),
|
||||
}
|
||||
h.log.Error("couldn't delete objects", fields...)
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, response); err != nil {
|
||||
h.logAndSendError(w, "could not write response", reqInfo, err)
|
||||
if err = api.EncodeToResponse(w, response); err != nil {
|
||||
h.logAndSendError(w, "could not write response", reqInfo, err, zap.Array("objects", marshaler))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
|
@ -268,7 +270,7 @@ func (h *handler) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
var sessionToken *session.Container
|
||||
|
||||
boxData, err := middleware.GetBoxData(r.Context())
|
||||
boxData, err := layer.GetBoxData(r.Context())
|
||||
if err == nil {
|
||||
sessionToken = boxData.Gate.SessionTokenForDelete()
|
||||
}
|
||||
|
@ -279,17 +281,5 @@ func (h *handler) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}); err != nil {
|
||||
h.logAndSendError(w, "couldn't delete bucket", reqInfo, err)
|
||||
}
|
||||
|
||||
chainIDs := []chain.ID{
|
||||
getBucketChainID(chain.S3, bktInfo),
|
||||
getBucketChainID(chain.Ingress, bktInfo),
|
||||
getBucketCannedChainID(chain.S3, bktInfo.CID),
|
||||
getBucketCannedChainID(chain.Ingress, bktInfo.CID),
|
||||
}
|
||||
if err = h.ape.DeleteBucketPolicy(reqInfo.Namespace, bktInfo.CID, chainIDs); err != nil {
|
||||
h.logAndSendError(w, "failed to delete policy from storage", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
|
|
@ -2,21 +2,12 @@ package handler
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -24,31 +15,6 @@ const (
|
|||
emptyVersion = ""
|
||||
)
|
||||
|
||||
func TestDeleteBucketOnAlreadyRemovedError(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
addr := getAddressOfLastVersion(hc, bktInfo, objName)
|
||||
hc.tp.SetObjectError(addr, &apistatus.ObjectAlreadyRemoved{})
|
||||
|
||||
deleteObjects(t, hc, bktName, [][2]string{{objName, emptyVersion}})
|
||||
|
||||
deleteBucket(t, hc, bktName, http.StatusNoContent)
|
||||
}
|
||||
|
||||
func getAddressOfLastVersion(hc *handlerContext, bktInfo *data.BucketInfo, objName string) oid.Address {
|
||||
nodeVersion, err := hc.tree.GetLatestVersion(hc.context, bktInfo, objName)
|
||||
require.NoError(hc.t, err)
|
||||
var addr oid.Address
|
||||
addr.SetContainer(bktInfo.CID)
|
||||
addr.SetObject(nodeVersion.OID)
|
||||
return addr
|
||||
}
|
||||
|
||||
func TestDeleteBucket(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
|
@ -65,58 +31,6 @@ func TestDeleteBucket(t *testing.T) {
|
|||
deleteBucket(t, tc, bktName, http.StatusNoContent)
|
||||
}
|
||||
|
||||
func TestDeleteBucketOnNotFoundError(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
nodeVersion, err := hc.tree.GetUnversioned(hc.context, bktInfo, objName)
|
||||
require.NoError(t, err)
|
||||
var addr oid.Address
|
||||
addr.SetContainer(bktInfo.CID)
|
||||
addr.SetObject(nodeVersion.OID)
|
||||
hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
|
||||
|
||||
deleteObjects(t, hc, bktName, [][2]string{{objName, emptyVersion}})
|
||||
|
||||
deleteBucket(t, hc, bktName, http.StatusNoContent)
|
||||
}
|
||||
|
||||
func TestDeleteObjectsError(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
putBucketVersioning(t, hc, bktName, true)
|
||||
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
nodeVersion, err := hc.tree.GetLatestVersion(hc.context, bktInfo, objName)
|
||||
require.NoError(t, err)
|
||||
var addr oid.Address
|
||||
addr.SetContainer(bktInfo.CID)
|
||||
addr.SetObject(nodeVersion.OID)
|
||||
|
||||
expectedError := apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
|
||||
hc.tp.SetObjectError(addr, expectedError)
|
||||
|
||||
w := deleteObjectsBase(hc, bktName, [][2]string{{objName, nodeVersion.OID.EncodeToString()}})
|
||||
|
||||
res := &s3.DeleteObjectsOutput{}
|
||||
err = xmlutil.UnmarshalXML(res, xml.NewDecoder(w.Result().Body), "")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.ElementsMatch(t, []*s3.Error{{
|
||||
Code: aws.String(expectedError.Code),
|
||||
Key: aws.String(objName),
|
||||
Message: aws.String(expectedError.Error()),
|
||||
VersionId: aws.String(nodeVersion.OID.EncodeToString()),
|
||||
}}, res.Errors)
|
||||
}
|
||||
|
||||
func TestDeleteObject(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
|
@ -135,7 +49,7 @@ func TestDeleteObjectFromSuspended(t *testing.T) {
|
|||
bktName, objName := "bucket-versioned-for-removal", "object-to-delete"
|
||||
|
||||
createSuspendedBucket(t, tc, bktName)
|
||||
putObject(tc, bktName, objName)
|
||||
putObject(t, tc, bktName, objName)
|
||||
|
||||
versionID, isDeleteMarker := deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||
require.True(t, isDeleteMarker)
|
||||
|
@ -168,7 +82,7 @@ func TestDeleteDeletedObject(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("versioned bucket not found obj", func(t *testing.T) {
|
||||
bktName, objName := "bucket-versioned-for-removal-not-found", "object-to-delete"
|
||||
bktName, objName := "bucket-versioned-for-removal", "object-to-delete"
|
||||
_, objInfo := createVersionedBucketAndObject(t, tc, bktName, objName)
|
||||
|
||||
versionID, isDeleteMarker := deleteObject(t, tc, bktName, objName, objInfo.VersionID())
|
||||
|
@ -233,82 +147,6 @@ func TestRemoveDeleteMarker(t *testing.T) {
|
|||
require.True(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object doesn't exist but should")
|
||||
}
|
||||
|
||||
func TestDeleteMarkerVersioned(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||
createVersionedBucketAndObject(t, tc, bktName, objName)
|
||||
|
||||
t.Run("not create new delete marker if last version is delete marker", func(t *testing.T) {
|
||||
deleteMarkerVersion, isDeleteMarker := deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||
require.True(t, isDeleteMarker)
|
||||
versions := listVersions(t, tc, bktName)
|
||||
require.Len(t, versions.DeleteMarker, 1)
|
||||
require.Equal(t, deleteMarkerVersion, versions.DeleteMarker[0].VersionID)
|
||||
|
||||
_, isDeleteMarker = deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||
require.True(t, isDeleteMarker)
|
||||
versions = listVersions(t, tc, bktName)
|
||||
require.Len(t, versions.DeleteMarker, 1)
|
||||
require.Equal(t, deleteMarkerVersion, versions.DeleteMarker[0].VersionID)
|
||||
})
|
||||
|
||||
t.Run("do not create delete marker if object does not exist", func(t *testing.T) {
|
||||
versionsBefore := listVersions(t, tc, bktName)
|
||||
_, isDeleteMarker := deleteObject(t, tc, bktName, "dummy", emptyVersion)
|
||||
require.False(t, isDeleteMarker)
|
||||
versionsAfter := listVersions(t, tc, bktName)
|
||||
require.Equal(t, versionsBefore, versionsAfter)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeleteMarkerSuspended(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||
bktInfo, _ := createVersionedBucketAndObject(t, tc, bktName, objName)
|
||||
putBucketVersioning(t, tc, bktName, false)
|
||||
|
||||
t.Run("not create new delete marker if last version is delete marker", func(t *testing.T) {
|
||||
deleteMarkerVersion, isDeleteMarker := deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||
require.True(t, isDeleteMarker)
|
||||
require.Equal(t, data.UnversionedObjectVersionID, deleteMarkerVersion)
|
||||
|
||||
deleteMarkerVersion, isDeleteMarker = deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||
require.True(t, isDeleteMarker)
|
||||
require.Equal(t, data.UnversionedObjectVersionID, deleteMarkerVersion)
|
||||
|
||||
versions := listVersions(t, tc, bktName)
|
||||
require.Len(t, versions.DeleteMarker, 1)
|
||||
require.Equal(t, deleteMarkerVersion, versions.DeleteMarker[0].VersionID)
|
||||
})
|
||||
|
||||
t.Run("do not create delete marker if object does not exist", func(t *testing.T) {
|
||||
versionsBefore := listVersions(t, tc, bktName)
|
||||
_, isDeleteMarker := deleteObject(t, tc, bktName, "dummy", emptyVersion)
|
||||
require.False(t, isDeleteMarker)
|
||||
versionsAfter := listVersions(t, tc, bktName)
|
||||
require.Equal(t, versionsBefore, versionsAfter)
|
||||
})
|
||||
|
||||
t.Run("remove last unversioned non delete marker", func(t *testing.T) {
|
||||
objName := "obj3"
|
||||
putObject(tc, bktName, objName)
|
||||
|
||||
nodeVersion, err := tc.tree.GetUnversioned(tc.Context(), bktInfo, objName)
|
||||
require.NoError(t, err)
|
||||
|
||||
deleteMarkerVersion, isDeleteMarker := deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||
require.True(t, isDeleteMarker)
|
||||
require.Equal(t, data.UnversionedObjectVersionID, deleteMarkerVersion)
|
||||
|
||||
objVersions := getVersion(listVersions(t, tc, bktName), objName)
|
||||
require.Len(t, objVersions, 0)
|
||||
|
||||
require.False(t, tc.MockedPool().ObjectExists(nodeVersion.OID))
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeleteObjectCombined(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
|
@ -359,40 +197,19 @@ func TestDeleteMarkers(t *testing.T) {
|
|||
deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||
|
||||
versions := listVersions(t, tc, bktName)
|
||||
require.Len(t, versions.DeleteMarker, 0, "invalid delete markers length")
|
||||
require.Len(t, versions.DeleteMarker, 3, "invalid delete markers length")
|
||||
require.Len(t, versions.Version, 0, "versions must be empty")
|
||||
|
||||
require.Len(t, listOIDsFromMockedFrostFS(t, tc, bktName), 0, "shouldn't be any object in frostfs")
|
||||
}
|
||||
|
||||
func TestGetHeadDeleteMarker(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||
createTestBucket(hc, bktName)
|
||||
putBucketVersioning(t, hc, bktName, true)
|
||||
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
deleteMarkerVersionID, _ := deleteObject(t, hc, bktName, objName, emptyVersion)
|
||||
|
||||
w := headObjectBase(hc, bktName, objName, deleteMarkerVersionID)
|
||||
require.Equal(t, w.Code, http.StatusMethodNotAllowed)
|
||||
require.Equal(t, w.Result().Header.Get(api.AmzDeleteMarker), "true")
|
||||
|
||||
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
||||
hc.Handler().GetObjectHandler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusNotFound)
|
||||
require.Equal(t, w.Result().Header.Get(api.AmzDeleteMarker), "true")
|
||||
}
|
||||
|
||||
func TestDeleteObjectFromListCache(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||
bktInfo, objInfo := createVersionedBucketAndObject(t, tc, bktName, objName)
|
||||
|
||||
versions := listObjectsV1(tc, bktName, "", "", "", -1)
|
||||
versions := listObjectsV1(t, tc, bktName, "", "", "", -1)
|
||||
require.Len(t, versions.Contents, 1)
|
||||
|
||||
checkFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||
|
@ -400,7 +217,7 @@ func TestDeleteObjectFromListCache(t *testing.T) {
|
|||
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||
|
||||
// check cache is clean after object removal
|
||||
versions = listObjectsV1(tc, bktName, "", "", "", -1)
|
||||
versions = listObjectsV1(t, tc, bktName, "", "", "", -1)
|
||||
require.Len(t, versions.Contents, 0)
|
||||
|
||||
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
|
||||
|
@ -429,25 +246,22 @@ func TestDeleteObjectCheckMarkerReturn(t *testing.T) {
|
|||
func createBucketAndObject(tc *handlerContext, bktName, objName string) (*data.BucketInfo, *data.ObjectInfo) {
|
||||
bktInfo := createTestBucket(tc, bktName)
|
||||
|
||||
objInfo := createTestObject(tc, bktInfo, objName, encryption.Params{})
|
||||
objInfo := createTestObject(tc, bktInfo, objName)
|
||||
|
||||
return bktInfo, objInfo
|
||||
}
|
||||
|
||||
func createVersionedBucketAndObject(_ *testing.T, tc *handlerContext, bktName, objName string) (*data.BucketInfo, *data.ObjectInfo) {
|
||||
bktInfo := createVersionedBucket(tc, bktName)
|
||||
objInfo := createTestObject(tc, bktInfo, objName, encryption.Params{})
|
||||
func createVersionedBucketAndObject(t *testing.T, tc *handlerContext, bktName, objName string) (*data.BucketInfo, *data.ObjectInfo) {
|
||||
createTestBucket(tc, bktName)
|
||||
bktInfo, err := tc.Layer().GetBucketInfo(tc.Context(), bktName)
|
||||
require.NoError(t, err)
|
||||
putBucketVersioning(t, tc, bktName, true)
|
||||
|
||||
objInfo := createTestObject(tc, bktInfo, objName)
|
||||
|
||||
return bktInfo, objInfo
|
||||
}
|
||||
|
||||
func createVersionedBucket(hc *handlerContext, bktName string) *data.BucketInfo {
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
putBucketVersioning(hc.t, hc, bktName, true)
|
||||
|
||||
return bktInfo
|
||||
}
|
||||
|
||||
func putBucketVersioning(t *testing.T, tc *handlerContext, bktName string, enabled bool) {
|
||||
cfg := &VersioningConfiguration{Status: "Suspended"}
|
||||
if enabled {
|
||||
|
@ -469,80 +283,44 @@ func deleteObject(t *testing.T, tc *handlerContext, bktName, objName, version st
|
|||
return w.Header().Get(api.AmzVersionID), w.Header().Get(api.AmzDeleteMarker) != ""
|
||||
}
|
||||
|
||||
func deleteObjects(t *testing.T, tc *handlerContext, bktName string, objVersions [][2]string) *DeleteObjectsResponse {
|
||||
w := deleteObjectsBase(tc, bktName, objVersions)
|
||||
|
||||
res := &DeleteObjectsResponse{}
|
||||
parseTestResponse(t, w, res)
|
||||
return res
|
||||
}
|
||||
|
||||
func deleteObjectsBase(hc *handlerContext, bktName string, objVersions [][2]string) *httptest.ResponseRecorder {
|
||||
req := &DeleteObjectsRequest{}
|
||||
for _, version := range objVersions {
|
||||
req.Objects = append(req.Objects, ObjectIdentifier{
|
||||
ObjectName: version[0],
|
||||
VersionID: version[1],
|
||||
})
|
||||
}
|
||||
|
||||
w, r := prepareTestRequest(hc, bktName, "", req)
|
||||
r.Header.Set(api.ContentMD5, "")
|
||||
hc.Handler().DeleteMultipleObjectsHandler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
|
||||
return w
|
||||
}
|
||||
|
||||
func deleteBucket(t *testing.T, tc *handlerContext, bktName string, code int) {
|
||||
w, r := prepareTestRequest(tc, bktName, "", nil)
|
||||
tc.Handler().DeleteBucketHandler(w, r)
|
||||
assertStatus(t, w, code)
|
||||
}
|
||||
|
||||
func checkNotFound(t *testing.T, hc *handlerContext, bktName, objName, version string) {
|
||||
w := headObjectBase(hc, bktName, objName, version)
|
||||
assertStatus(t, w, http.StatusNotFound)
|
||||
}
|
||||
|
||||
func headObjectAssertS3Error(hc *handlerContext, bktName, objName, version string, code apiErrors.ErrorCode) {
|
||||
w := headObjectBase(hc, bktName, objName, version)
|
||||
assertS3Error(hc.t, w, apiErrors.GetAPIError(code))
|
||||
}
|
||||
|
||||
func checkFound(t *testing.T, hc *handlerContext, bktName, objName, version string) {
|
||||
w := headObjectBase(hc, bktName, objName, version)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func headObjectBase(hc *handlerContext, bktName, objName, version string) *httptest.ResponseRecorder {
|
||||
func checkNotFound(t *testing.T, tc *handlerContext, bktName, objName, version string) {
|
||||
query := make(url.Values)
|
||||
query.Add(api.QueryVersionID, version)
|
||||
|
||||
w, r := prepareTestFullRequest(hc, bktName, objName, query, nil)
|
||||
hc.Handler().HeadObjectHandler(w, r)
|
||||
return w
|
||||
w, r := prepareTestFullRequest(tc, bktName, objName, query, nil)
|
||||
tc.Handler().HeadObjectHandler(w, r)
|
||||
assertStatus(t, w, http.StatusNotFound)
|
||||
}
|
||||
|
||||
func listVersions(_ *testing.T, tc *handlerContext, bktName string) *ListObjectsVersionsResponse {
|
||||
return listObjectsVersions(tc, bktName, "", "", "", "", -1)
|
||||
func checkFound(t *testing.T, tc *handlerContext, bktName, objName, version string) {
|
||||
query := make(url.Values)
|
||||
query.Add(api.QueryVersionID, version)
|
||||
|
||||
w, r := prepareTestFullRequest(tc, bktName, objName, query, nil)
|
||||
tc.Handler().HeadObjectHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func getVersion(resp *ListObjectsVersionsResponse, objName string) []*ObjectVersionResponse {
|
||||
var res []*ObjectVersionResponse
|
||||
for i, version := range resp.Version {
|
||||
if version.Key == objName {
|
||||
res = append(res, &resp.Version[i])
|
||||
}
|
||||
}
|
||||
func listVersions(t *testing.T, tc *handlerContext, bktName string) *ListObjectsVersionsResponse {
|
||||
w, r := prepareTestRequest(tc, bktName, "", nil)
|
||||
tc.Handler().ListBucketObjectVersionsHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
res := &ListObjectsVersionsResponse{}
|
||||
parseTestResponse(t, w, res)
|
||||
return res
|
||||
}
|
||||
|
||||
func putObject(hc *handlerContext, bktName, objName string) {
|
||||
func putObject(t *testing.T, tc *handlerContext, bktName, objName string) {
|
||||
body := bytes.NewReader([]byte("content"))
|
||||
w, r := prepareTestPayloadRequest(hc, bktName, objName, body)
|
||||
hc.Handler().PutObjectHandler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, body)
|
||||
tc.Handler().PutObjectHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func createSuspendedBucket(t *testing.T, tc *handlerContext, bktName string) *data.BucketInfo {
|
||||
|
|
|
@ -7,15 +7,13 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -43,7 +41,7 @@ func TestSimpleGetEncrypted(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.NotEqual(t, content, string(encryptedContent))
|
||||
|
||||
response, _ := getEncryptedObject(tc, bktName, objName)
|
||||
response, _ := getEncryptedObject(t, tc, bktName, objName)
|
||||
require.Equal(t, content, string(response))
|
||||
}
|
||||
|
||||
|
@ -105,40 +103,14 @@ func TestS3EncryptionSSECMultipartUpload(t *testing.T) {
|
|||
data := multipartUploadEncrypted(tc, bktName, objName, headers, objLen, partSize)
|
||||
require.Equal(t, objLen, len(data))
|
||||
|
||||
resData, resHeader := getEncryptedObject(tc, bktName, objName)
|
||||
resData, resHeader := getEncryptedObject(t, tc, bktName, objName)
|
||||
equalDataSlices(t, data, resData)
|
||||
require.Equal(t, headers[api.ContentType], resHeader.Get(api.ContentType))
|
||||
require.Equal(t, headers[headerMetaKey], resHeader[headerMetaKey][0])
|
||||
require.Equal(t, strconv.Itoa(objLen), resHeader.Get(api.ContentLength))
|
||||
|
||||
checkContentUsingRangeEnc(tc, bktName, objName, data, 1000000)
|
||||
checkContentUsingRangeEnc(tc, bktName, objName, data, 10000000)
|
||||
}
|
||||
|
||||
func TestMultipartUploadGetRange(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName, objName := "bucket-for-multipart-s3-tests", "multipart_obj"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objLen := 30 * 1024 * 1024
|
||||
partSize := objLen / 6
|
||||
headerMetaKey := api.MetadataPrefix + "foo"
|
||||
headers := map[string]string{
|
||||
headerMetaKey: "bar",
|
||||
api.ContentType: "text/plain",
|
||||
}
|
||||
|
||||
data := multipartUpload(hc, bktName, objName, headers, objLen, partSize)
|
||||
require.Equal(t, objLen, len(data))
|
||||
|
||||
resData, resHeader := getObject(hc, bktName, objName)
|
||||
equalDataSlices(t, data, resData)
|
||||
require.Equal(t, headers[api.ContentType], resHeader.Get(api.ContentType))
|
||||
require.Equal(t, headers[headerMetaKey], resHeader[headerMetaKey][0])
|
||||
require.Equal(t, strconv.Itoa(objLen), resHeader.Get(api.ContentLength))
|
||||
|
||||
checkContentUsingRange(hc, bktName, objName, data, 1000000)
|
||||
checkContentUsingRange(hc, bktName, objName, data, 10000000)
|
||||
checkContentUsingRangeEnc(t, tc, bktName, objName, data, 1000000)
|
||||
checkContentUsingRangeEnc(t, tc, bktName, objName, data, 10000000)
|
||||
}
|
||||
|
||||
func equalDataSlices(t *testing.T, expected, actual []byte) {
|
||||
|
@ -155,15 +127,7 @@ func equalDataSlices(t *testing.T, expected, actual []byte) {
|
|||
}
|
||||
}
|
||||
|
||||
func checkContentUsingRangeEnc(hc *handlerContext, bktName, objName string, data []byte, step int) {
|
||||
checkContentUsingRangeBase(hc, bktName, objName, data, step, true)
|
||||
}
|
||||
|
||||
func checkContentUsingRange(hc *handlerContext, bktName, objName string, data []byte, step int) {
|
||||
checkContentUsingRangeBase(hc, bktName, objName, data, step, false)
|
||||
}
|
||||
|
||||
func checkContentUsingRangeBase(hc *handlerContext, bktName, objName string, data []byte, step int, encrypted bool) {
|
||||
func checkContentUsingRangeEnc(t *testing.T, tc *handlerContext, bktName, objName string, data []byte, step int) {
|
||||
var off, toRead, end int
|
||||
|
||||
for off < len(data) {
|
||||
|
@ -173,14 +137,8 @@ func checkContentUsingRangeBase(hc *handlerContext, bktName, objName string, dat
|
|||
}
|
||||
end = off + toRead - 1
|
||||
|
||||
var rangeData []byte
|
||||
if encrypted {
|
||||
rangeData = getEncryptedObjectRange(hc.t, hc, bktName, objName, off, end)
|
||||
} else {
|
||||
rangeData = getObjectRange(hc.t, hc, bktName, objName, off, end)
|
||||
}
|
||||
|
||||
equalDataSlices(hc.t, data[off:end+1], rangeData)
|
||||
rangeData := getEncryptedObjectRange(t, tc, bktName, objName, off, end)
|
||||
equalDataSlices(t, data[off:end+1], rangeData)
|
||||
|
||||
off += step
|
||||
}
|
||||
|
@ -210,66 +168,28 @@ func multipartUploadEncrypted(hc *handlerContext, bktName, objName string, heade
|
|||
return
|
||||
}
|
||||
|
||||
func multipartUpload(hc *handlerContext, bktName, objName string, headers map[string]string, objLen, partsSize int) (objData []byte) {
|
||||
multipartInfo := createMultipartUpload(hc, bktName, objName, headers)
|
||||
|
||||
var sum, currentPart int
|
||||
var etags []string
|
||||
adjustedSize := partsSize
|
||||
|
||||
for sum < objLen {
|
||||
currentPart++
|
||||
|
||||
sum += partsSize
|
||||
if sum > objLen {
|
||||
adjustedSize = objLen - sum
|
||||
}
|
||||
|
||||
etag, data := uploadPart(hc, bktName, objName, multipartInfo.UploadID, currentPart, adjustedSize)
|
||||
etags = append(etags, etag)
|
||||
objData = append(objData, data...)
|
||||
}
|
||||
|
||||
completeMultipartUpload(hc, bktName, objName, multipartInfo.UploadID, etags)
|
||||
return
|
||||
}
|
||||
|
||||
func createMultipartUploadEncrypted(hc *handlerContext, bktName, objName string, headers map[string]string) *InitiateMultipartUploadResponse {
|
||||
return createMultipartUploadOkBase(hc, bktName, objName, true, headers)
|
||||
return createMultipartUploadBase(hc, bktName, objName, true, headers)
|
||||
}
|
||||
|
||||
func createMultipartUpload(hc *handlerContext, bktName, objName string, headers map[string]string) *InitiateMultipartUploadResponse {
|
||||
return createMultipartUploadOkBase(hc, bktName, objName, false, headers)
|
||||
return createMultipartUploadBase(hc, bktName, objName, false, headers)
|
||||
}
|
||||
|
||||
func createMultipartUploadOkBase(hc *handlerContext, bktName, objName string, encrypted bool, headers map[string]string) *InitiateMultipartUploadResponse {
|
||||
w := createMultipartUploadBase(hc, bktName, objName, encrypted, headers)
|
||||
multipartInitInfo := &InitiateMultipartUploadResponse{}
|
||||
readResponse(hc.t, w, http.StatusOK, multipartInitInfo)
|
||||
return multipartInitInfo
|
||||
}
|
||||
|
||||
func createMultipartUploadAssertS3Error(hc *handlerContext, bktName, objName string, headers map[string]string, code errors.ErrorCode) {
|
||||
w := createMultipartUploadBase(hc, bktName, objName, false, headers)
|
||||
assertS3Error(hc.t, w, errors.GetAPIError(code))
|
||||
}
|
||||
|
||||
func createMultipartUploadBase(hc *handlerContext, bktName, objName string, encrypted bool, headers map[string]string) *httptest.ResponseRecorder {
|
||||
func createMultipartUploadBase(hc *handlerContext, bktName, objName string, encrypted bool, headers map[string]string) *InitiateMultipartUploadResponse {
|
||||
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
||||
if encrypted {
|
||||
setEncryptHeaders(r)
|
||||
}
|
||||
setHeaders(r, headers)
|
||||
hc.Handler().CreateMultipartUploadHandler(w, r)
|
||||
return w
|
||||
multipartInitInfo := &InitiateMultipartUploadResponse{}
|
||||
readResponse(hc.t, w, http.StatusOK, multipartInitInfo)
|
||||
|
||||
return multipartInitInfo
|
||||
}
|
||||
|
||||
func completeMultipartUpload(hc *handlerContext, bktName, objName, uploadID string, partsETags []string) {
|
||||
w := completeMultipartUploadBase(hc, bktName, objName, uploadID, partsETags)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func completeMultipartUploadBase(hc *handlerContext, bktName, objName, uploadID string, partsETags []string) *httptest.ResponseRecorder {
|
||||
query := make(url.Values)
|
||||
query.Set(uploadIDQuery, uploadID)
|
||||
complete := &CompleteMultipartUpload{
|
||||
|
@ -284,8 +204,7 @@ func completeMultipartUploadBase(hc *handlerContext, bktName, objName, uploadID
|
|||
|
||||
w, r := prepareTestFullRequest(hc, bktName, objName, query, complete)
|
||||
hc.Handler().CompleteMultipartUploadHandler(w, r)
|
||||
|
||||
return w
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func uploadPartEncrypted(hc *handlerContext, bktName, objName, uploadID string, num, size int) (string, []byte) {
|
||||
|
@ -328,7 +247,7 @@ func TestMultipartEncrypted(t *testing.T) {
|
|||
part2ETag, part2 := uploadPartEncrypted(hc, bktName, objName, multipartInitInfo.UploadID, 2, 5)
|
||||
completeMultipartUpload(hc, bktName, objName, multipartInitInfo.UploadID, []string{part1ETag, part2ETag})
|
||||
|
||||
res, _ := getEncryptedObject(hc, bktName, objName)
|
||||
res, _ := getEncryptedObject(t, hc, bktName, objName)
|
||||
require.Equal(t, len(part1)+len(part2), len(res))
|
||||
require.Equal(t, append(part1, part2...), res)
|
||||
|
||||
|
@ -344,22 +263,13 @@ func putEncryptedObject(t *testing.T, tc *handlerContext, bktName, objName, cont
|
|||
assertStatus(t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func getEncryptedObject(hc *handlerContext, bktName, objName string) ([]byte, http.Header) {
|
||||
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
||||
func getEncryptedObject(t *testing.T, tc *handlerContext, bktName, objName string) ([]byte, http.Header) {
|
||||
w, r := prepareTestRequest(tc, bktName, objName, nil)
|
||||
setEncryptHeaders(r)
|
||||
return getObjectBase(hc, w, r)
|
||||
}
|
||||
|
||||
func getObject(hc *handlerContext, bktName, objName string) ([]byte, http.Header) {
|
||||
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
||||
return getObjectBase(hc, w, r)
|
||||
}
|
||||
|
||||
func getObjectBase(hc *handlerContext, w *httptest.ResponseRecorder, r *http.Request) ([]byte, http.Header) {
|
||||
hc.Handler().GetObjectHandler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
tc.Handler().GetObjectHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
content, err := io.ReadAll(w.Result().Body)
|
||||
require.NoError(hc.t, err)
|
||||
require.NoError(t, err)
|
||||
return content, w.Header()
|
||||
}
|
||||
|
||||
|
|
|
@ -8,11 +8,10 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -78,8 +77,7 @@ func addSSECHeaders(responseHeader http.Header, requestHeader http.Header) {
|
|||
responseHeader.Set(api.AmzServerSideEncryptionCustomerKeyMD5, requestHeader.Get(api.AmzServerSideEncryptionCustomerKeyMD5))
|
||||
}
|
||||
|
||||
func writeHeaders(h http.Header, requestHeader http.Header, extendedInfo *data.ExtendedObjectInfo, tagSetLength int,
|
||||
isBucketUnversioned, md5Enabled bool) {
|
||||
func writeHeaders(h http.Header, requestHeader http.Header, extendedInfo *data.ExtendedObjectInfo, tagSetLength int, isBucketUnversioned bool) {
|
||||
info := extendedInfo.ObjectInfo
|
||||
if len(info.ContentType) > 0 && h.Get(api.ContentType) == "" {
|
||||
h.Set(api.ContentType, info.ContentType)
|
||||
|
@ -89,16 +87,12 @@ func writeHeaders(h http.Header, requestHeader http.Header, extendedInfo *data.E
|
|||
if len(info.Headers[layer.AttributeEncryptionAlgorithm]) > 0 {
|
||||
h.Set(api.ContentLength, info.Headers[layer.AttributeDecryptedSize])
|
||||
addSSECHeaders(h, requestHeader)
|
||||
} else if len(info.Headers[layer.MultipartObjectSize]) > 0 {
|
||||
h.Set(api.ContentLength, info.Headers[layer.MultipartObjectSize])
|
||||
} else {
|
||||
h.Set(api.ContentLength, strconv.FormatUint(info.Size, 10))
|
||||
h.Set(api.ContentLength, strconv.FormatInt(info.Size, 10))
|
||||
}
|
||||
|
||||
h.Set(api.ETag, data.Quote(info.ETag(md5Enabled)))
|
||||
|
||||
h.Set(api.ETag, info.HashSum)
|
||||
h.Set(api.AmzTaggingCount, strconv.Itoa(tagSetLength))
|
||||
h.Set(api.AmzStorageClass, api.DefaultStorageClass)
|
||||
|
||||
if !isBucketUnversioned {
|
||||
h.Set(api.AmzVersionID, extendedInfo.Version())
|
||||
|
@ -110,12 +104,6 @@ func writeHeaders(h http.Header, requestHeader http.Header, extendedInfo *data.E
|
|||
if expires := info.Headers[api.Expires]; expires != "" {
|
||||
h.Set(api.Expires, expires)
|
||||
}
|
||||
if encodings := info.Headers[api.ContentEncoding]; encodings != "" {
|
||||
h.Set(api.ContentEncoding, encodings)
|
||||
}
|
||||
if contentLanguage := info.Headers[api.ContentLanguage]; contentLanguage != "" {
|
||||
h.Set(api.ContentLanguage, contentLanguage)
|
||||
}
|
||||
|
||||
for key, val := range info.Headers {
|
||||
if layer.IsSystemHeader(key) {
|
||||
|
@ -129,7 +117,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
var (
|
||||
params *layer.RangeParams
|
||||
|
||||
reqInfo = middleware.GetReqInfo(r.Context())
|
||||
reqInfo = api.GetReqInfo(r.Context())
|
||||
)
|
||||
|
||||
conditional, err := parseConditionalHeaders(r.Header)
|
||||
|
@ -157,7 +145,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
info := extendedInfo.ObjectInfo
|
||||
|
||||
if err = checkPreconditions(info, conditional, h.cfg.MD5Enabled()); err != nil {
|
||||
if err = checkPreconditions(info, conditional); err != nil {
|
||||
h.logAndSendError(w, "precondition failed", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
@ -173,18 +161,20 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
fullSize, err := layer.GetObjectSize(info)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid size header", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
|
||||
return
|
||||
fullSize := info.Size
|
||||
if encryptionParams.Enabled() {
|
||||
if fullSize, err = strconv.ParseInt(info.Headers[layer.AttributeDecryptedSize], 10, 64); err != nil {
|
||||
h.logAndSendError(w, "invalid decrypted size header", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if params, err = fetchRangeHeader(r.Header, fullSize); err != nil {
|
||||
if params, err = fetchRangeHeader(r.Header, uint64(fullSize)); err != nil {
|
||||
h.logAndSendError(w, "could not parse range header", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
t := &data.ObjectVersion{
|
||||
t := &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: info.Name,
|
||||
VersionID: info.VersionID(),
|
||||
|
@ -211,54 +201,38 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
getPayloadParams := &layer.GetObjectParams{
|
||||
ObjectInfo: info,
|
||||
Versioned: p.Versioned(),
|
||||
Range: params,
|
||||
BucketInfo: bktInfo,
|
||||
Encryption: encryptionParams,
|
||||
}
|
||||
|
||||
objPayload, err := h.obj.GetObject(r.Context(), getPayloadParams)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get object payload", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
writeHeaders(w.Header(), r.Header, extendedInfo, len(tagSet), bktSettings.Unversioned(), h.cfg.MD5Enabled())
|
||||
writeHeaders(w.Header(), r.Header, extendedInfo, len(tagSet), bktSettings.Unversioned())
|
||||
if params != nil {
|
||||
writeRangeHeaders(w, params, fullSize)
|
||||
writeRangeHeaders(w, params, info.Size)
|
||||
} else {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
if err = objPayload.StreamTo(w); err != nil {
|
||||
h.logAndSendError(w, "could not stream object payload", reqInfo, err)
|
||||
return
|
||||
getParams := &layer.GetObjectParams{
|
||||
ObjectInfo: info,
|
||||
Writer: w,
|
||||
Range: params,
|
||||
BucketInfo: bktInfo,
|
||||
Encryption: encryptionParams,
|
||||
}
|
||||
if err = h.obj.GetObject(r.Context(), getParams); err != nil {
|
||||
h.logAndSendError(w, "could not get object", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs, md5Enabled bool) error {
|
||||
etag := info.ETag(md5Enabled)
|
||||
if len(args.IfMatch) > 0 && args.IfMatch != etag {
|
||||
return fmt.Errorf("%w: etag mismatched: '%s', '%s'", errors.GetAPIError(errors.ErrPreconditionFailed), args.IfMatch, etag)
|
||||
func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs) error {
|
||||
if len(args.IfMatch) > 0 && args.IfMatch != info.HashSum {
|
||||
return errors.GetAPIError(errors.ErrPreconditionFailed)
|
||||
}
|
||||
if len(args.IfNoneMatch) > 0 && args.IfNoneMatch == etag {
|
||||
return fmt.Errorf("%w: etag matched: '%s', '%s'", errors.GetAPIError(errors.ErrNotModified), args.IfNoneMatch, etag)
|
||||
if len(args.IfNoneMatch) > 0 && args.IfNoneMatch == info.HashSum {
|
||||
return errors.GetAPIError(errors.ErrNotModified)
|
||||
}
|
||||
if args.IfModifiedSince != nil && info.Created.Before(*args.IfModifiedSince) {
|
||||
return fmt.Errorf("%w: not modified since '%s', last modified '%s'", errors.GetAPIError(errors.ErrNotModified),
|
||||
args.IfModifiedSince.Format(time.RFC3339), info.Created.Format(time.RFC3339))
|
||||
return errors.GetAPIError(errors.ErrNotModified)
|
||||
}
|
||||
if args.IfUnmodifiedSince != nil && info.Created.After(*args.IfUnmodifiedSince) {
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax
|
||||
// If both of the If-Match and If-Unmodified-Since headers are present in the request as follows:
|
||||
// If-Match condition evaluates to true, and;
|
||||
// If-Unmodified-Since condition evaluates to false;
|
||||
// then, S3 returns 200 OK and the data requested.
|
||||
if len(args.IfMatch) == 0 {
|
||||
return fmt.Errorf("%w: modified since '%s', last modified '%s'", errors.GetAPIError(errors.ErrPreconditionFailed),
|
||||
args.IfUnmodifiedSince.Format(time.RFC3339), info.Created.Format(time.RFC3339))
|
||||
return errors.GetAPIError(errors.ErrPreconditionFailed)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -268,8 +242,8 @@ func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs, md5Enabled
|
|||
func parseConditionalHeaders(headers http.Header) (*conditionalArgs, error) {
|
||||
var err error
|
||||
args := &conditionalArgs{
|
||||
IfMatch: data.UnQuote(headers.Get(api.IfMatch)),
|
||||
IfNoneMatch: data.UnQuote(headers.Get(api.IfNoneMatch)),
|
||||
IfMatch: headers.Get(api.IfMatch),
|
||||
IfNoneMatch: headers.Get(api.IfNoneMatch),
|
||||
}
|
||||
|
||||
if args.IfModifiedSince, err = parseHTTPTime(headers.Get(api.IfModifiedSince)); err != nil {
|
||||
|
@ -294,7 +268,7 @@ func parseHTTPTime(data string) (*time.Time, error) {
|
|||
return &result, nil
|
||||
}
|
||||
|
||||
func writeRangeHeaders(w http.ResponseWriter, params *layer.RangeParams, size uint64) {
|
||||
func writeRangeHeaders(w http.ResponseWriter, params *layer.RangeParams, size int64) {
|
||||
w.Header().Set(api.AcceptRanges, "bytes")
|
||||
w.Header().Set(api.ContentRange, fmt.Sprintf("bytes %d-%d/%d", params.Start, params.End, size))
|
||||
w.Header().Set(api.ContentLength, strconv.FormatUint(params.End-params.Start+1, 10))
|
||||
|
|
|
@ -2,20 +2,15 @@ package handler
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -147,12 +142,8 @@ func TestPreconditions(t *testing.T) {
|
|||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
actual := checkPreconditions(tc.info, tc.args, false)
|
||||
if tc.expected == nil {
|
||||
require.NoError(t, actual)
|
||||
} else {
|
||||
require.True(t, stderrors.Is(actual, tc.expected), tc.expected, actual)
|
||||
}
|
||||
actual := checkPreconditions(tc.info, tc.args)
|
||||
require.Equal(t, tc.expected, actual)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -179,43 +170,11 @@ func TestGetRange(t *testing.T) {
|
|||
require.Equal(t, "bcdef", string(end))
|
||||
}
|
||||
|
||||
func TestGetObject(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName, objName := "bucket", "obj"
|
||||
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
|
||||
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
|
||||
checkFound(hc.t, hc, bktName, objName, emptyVersion)
|
||||
|
||||
addr := getAddressOfLastVersion(hc, bktInfo, objName)
|
||||
hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
|
||||
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectNotFound{})
|
||||
|
||||
getObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), errors.ErrNoSuchVersion)
|
||||
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, errors.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
func TestGetObjectEnabledMD5(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName, objName := "bucket", "obj"
|
||||
_, objInfo := createBucketAndObject(hc, bktName, objName)
|
||||
|
||||
_, headers := getObject(hc, bktName, objName)
|
||||
require.Equal(t, data.Quote(objInfo.HashSum), headers.Get(api.ETag))
|
||||
|
||||
hc.config.md5Enabled = true
|
||||
_, headers = getObject(hc, bktName, objName)
|
||||
require.Equal(t, data.Quote(objInfo.MD5Sum), headers.Get(api.ETag))
|
||||
}
|
||||
|
||||
func putObjectContent(hc *handlerContext, bktName, objName, content string) http.Header {
|
||||
func putObjectContent(hc *handlerContext, bktName, objName, content string) {
|
||||
body := bytes.NewReader([]byte(content))
|
||||
w, r := prepareTestPayloadRequest(hc, bktName, objName, body)
|
||||
hc.Handler().PutObjectHandler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
return w.Result().Header
|
||||
}
|
||||
|
||||
func getObjectRange(t *testing.T, tc *handlerContext, bktName, objName string, start, end int) []byte {
|
||||
|
@ -227,17 +186,3 @@ func getObjectRange(t *testing.T, tc *handlerContext, bktName, objName string, s
|
|||
require.NoError(t, err)
|
||||
return content
|
||||
}
|
||||
|
||||
func getObjectAssertS3Error(hc *handlerContext, bktName, objName, version string, code errors.ErrorCode) {
|
||||
w := getObjectBaseResponse(hc, bktName, objName, version)
|
||||
assertS3Error(hc.t, w, errors.GetAPIError(code))
|
||||
}
|
||||
|
||||
func getObjectBaseResponse(hc *handlerContext, bktName, objName, version string) *httptest.ResponseRecorder {
|
||||
query := make(url.Values)
|
||||
query.Add(api.QueryVersionID, version)
|
||||
|
||||
w, r := prepareTestFullRequest(hc, bktName, objName, query, nil)
|
||||
hc.Handler().GetObjectHandler(w, r)
|
||||
return w
|
||||
}
|
||||
|
|
|
@ -3,12 +3,9 @@ package handler
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
|
@ -16,24 +13,18 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
type handlerContext struct {
|
||||
|
@ -41,13 +32,7 @@ type handlerContext struct {
|
|||
t *testing.T
|
||||
h *handler
|
||||
tp *layer.TestFrostFS
|
||||
tree *tree.Tree
|
||||
context context.Context
|
||||
config *configMock
|
||||
|
||||
layerFeatures *layer.FeatureSettingsMock
|
||||
treeMock *tree.ServiceClientMemory
|
||||
cache *layer.Cache
|
||||
}
|
||||
|
||||
func (hc *handlerContext) Handler() *handler {
|
||||
|
@ -66,86 +51,24 @@ func (hc *handlerContext) Context() context.Context {
|
|||
return hc.context
|
||||
}
|
||||
|
||||
type configMock struct {
|
||||
defaultPolicy netmap.PlacementPolicy
|
||||
copiesNumbers map[string][]uint32
|
||||
defaultCopiesNumbers []uint32
|
||||
bypassContentEncodingInChunks bool
|
||||
md5Enabled bool
|
||||
aclEnabled bool
|
||||
type placementPolicyMock struct {
|
||||
defaultPolicy netmap.PlacementPolicy
|
||||
}
|
||||
|
||||
func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy {
|
||||
return c.defaultPolicy
|
||||
func (p *placementPolicyMock) Default() netmap.PlacementPolicy {
|
||||
return p.defaultPolicy
|
||||
}
|
||||
|
||||
func (c *configMock) PlacementPolicy(_, _ string) (netmap.PlacementPolicy, bool) {
|
||||
func (p *placementPolicyMock) Get(string) (netmap.PlacementPolicy, bool) {
|
||||
return netmap.PlacementPolicy{}, false
|
||||
}
|
||||
|
||||
func (c *configMock) CopiesNumbers(_, locationConstraint string) ([]uint32, bool) {
|
||||
result, ok := c.copiesNumbers[locationConstraint]
|
||||
return result, ok
|
||||
}
|
||||
|
||||
func (c *configMock) DefaultCopiesNumbers(_ string) []uint32 {
|
||||
return c.defaultCopiesNumbers
|
||||
}
|
||||
|
||||
func (c *configMock) NewXMLDecoder(r io.Reader) *xml.Decoder {
|
||||
return xml.NewDecoder(r)
|
||||
}
|
||||
|
||||
func (c *configMock) BypassContentEncodingInChunks() bool {
|
||||
return c.bypassContentEncodingInChunks
|
||||
}
|
||||
|
||||
func (c *configMock) DefaultMaxAge() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (c *configMock) NotificatorEnabled() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *configMock) ResolveZoneList() []string {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
func (c *configMock) IsResolveListAllow() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *configMock) CompleteMultipartKeepalive() time.Duration {
|
||||
return time.Duration(0)
|
||||
}
|
||||
|
||||
func (c *configMock) MD5Enabled() bool {
|
||||
return c.md5Enabled
|
||||
}
|
||||
|
||||
func (c *configMock) ACLEnabled() bool {
|
||||
return c.aclEnabled
|
||||
}
|
||||
|
||||
func (c *configMock) ResolveNamespaceAlias(ns string) string {
|
||||
return ns
|
||||
}
|
||||
|
||||
func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||
return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(zap.NewExample()))
|
||||
}
|
||||
|
||||
func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
|
||||
return prepareHandlerContextBase(t, getMinCacheConfig(zap.NewExample()))
|
||||
}
|
||||
|
||||
func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *handlerContext {
|
||||
key, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
l := zap.NewExample()
|
||||
tp := layer.NewTestFrostFS(key)
|
||||
tp := layer.NewTestFrostFS()
|
||||
|
||||
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
||||
testResolver.SetResolveFunc(func(_ context.Context, name string) (cid.ID, error) {
|
||||
|
@ -155,35 +78,23 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand
|
|||
var owner user.ID
|
||||
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
|
||||
|
||||
memCli, err := tree.NewTreeServiceClientMemory()
|
||||
require.NoError(t, err)
|
||||
|
||||
treeMock := tree.NewTree(memCli, zap.NewExample())
|
||||
|
||||
features := &layer.FeatureSettingsMock{}
|
||||
|
||||
layerCfg := &layer.Config{
|
||||
Cache: layer.NewCache(cacheCfg),
|
||||
Caches: layer.DefaultCachesConfigs(zap.NewExample()),
|
||||
AnonKey: layer.AnonymousKey{Key: key},
|
||||
Resolver: testResolver,
|
||||
TreeService: treeMock,
|
||||
Features: features,
|
||||
GateOwner: owner,
|
||||
TreeService: layer.NewTreeService(),
|
||||
}
|
||||
|
||||
var pp netmap.PlacementPolicy
|
||||
err = pp.DecodeString("REP 1")
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg := &configMock{
|
||||
defaultPolicy: pp,
|
||||
}
|
||||
h := &handler{
|
||||
log: l,
|
||||
obj: layer.NewLayer(l, tp, layerCfg),
|
||||
cfg: cfg,
|
||||
ape: newAPEMock(),
|
||||
frostfsid: newFrostfsIDMock(),
|
||||
log: l,
|
||||
obj: layer.NewLayer(l, tp, layerCfg),
|
||||
cfg: &Config{
|
||||
Policy: &placementPolicyMock{defaultPolicy: pp},
|
||||
},
|
||||
}
|
||||
|
||||
return &handlerContext{
|
||||
|
@ -191,158 +102,24 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand
|
|||
t: t,
|
||||
h: h,
|
||||
tp: tp,
|
||||
tree: treeMock,
|
||||
context: middleware.SetBox(context.Background(), &middleware.Box{AccessBox: newTestAccessBox(t, key)}),
|
||||
config: cfg,
|
||||
|
||||
layerFeatures: features,
|
||||
treeMock: memCli,
|
||||
cache: layerCfg.Cache,
|
||||
context: context.WithValue(context.Background(), api.BoxData, newTestAccessBox(t, key)),
|
||||
}
|
||||
}
|
||||
|
||||
func getMinCacheConfig(logger *zap.Logger) *layer.CachesConfig {
|
||||
minCacheCfg := &cache.Config{
|
||||
Size: 1,
|
||||
Lifetime: 1,
|
||||
Logger: logger,
|
||||
}
|
||||
return &layer.CachesConfig{
|
||||
Logger: logger,
|
||||
Objects: minCacheCfg,
|
||||
ObjectsList: minCacheCfg,
|
||||
SessionList: minCacheCfg,
|
||||
Names: minCacheCfg,
|
||||
Buckets: minCacheCfg,
|
||||
System: minCacheCfg,
|
||||
AccessControl: minCacheCfg,
|
||||
}
|
||||
}
|
||||
|
||||
type apeMock struct {
|
||||
chainMap map[engine.Target][]*chain.Chain
|
||||
policyMap map[string][]byte
|
||||
}
|
||||
|
||||
func newAPEMock() *apeMock {
|
||||
return &apeMock{
|
||||
chainMap: map[engine.Target][]*chain.Chain{},
|
||||
policyMap: map[string][]byte{},
|
||||
}
|
||||
}
|
||||
|
||||
func (a *apeMock) AddChain(target engine.Target, c *chain.Chain) error {
|
||||
list := a.chainMap[target]
|
||||
|
||||
ind := slices.IndexFunc(list, func(item *chain.Chain) bool { return bytes.Equal(item.ID, c.ID) })
|
||||
if ind != -1 {
|
||||
list[ind] = c
|
||||
} else {
|
||||
list = append(list, c)
|
||||
}
|
||||
|
||||
a.chainMap[target] = list
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *apeMock) RemoveChain(target engine.Target, chainID chain.ID) error {
|
||||
a.chainMap[target] = slices.DeleteFunc(a.chainMap[target], func(item *chain.Chain) bool { return bytes.Equal(item.ID, chainID) })
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *apeMock) ListChains(target engine.Target) ([]*chain.Chain, error) {
|
||||
return a.chainMap[target], nil
|
||||
}
|
||||
|
||||
func (a *apeMock) PutPolicy(namespace string, cnrID cid.ID, policy []byte) error {
|
||||
a.policyMap[namespace+cnrID.EncodeToString()] = policy
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *apeMock) DeletePolicy(namespace string, cnrID cid.ID) error {
|
||||
delete(a.policyMap, namespace+cnrID.EncodeToString())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *apeMock) PutBucketPolicy(ns string, cnrID cid.ID, policy []byte, chain []*chain.Chain) error {
|
||||
if err := a.PutPolicy(ns, cnrID, policy); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := range chain {
|
||||
if err := a.AddChain(engine.ContainerTarget(cnrID.EncodeToString()), chain[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *apeMock) DeleteBucketPolicy(ns string, cnrID cid.ID, chainIDs []chain.ID) error {
|
||||
if err := a.DeletePolicy(ns, cnrID); err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range chainIDs {
|
||||
if err := a.RemoveChain(engine.ContainerTarget(cnrID.EncodeToString()), chainIDs[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *apeMock) GetBucketPolicy(ns string, cnrID cid.ID) ([]byte, error) {
|
||||
policy, ok := a.policyMap[ns+cnrID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, errors.New("not found")
|
||||
}
|
||||
|
||||
return policy, nil
|
||||
}
|
||||
|
||||
func (a *apeMock) SaveACLChains(cid string, chains []*chain.Chain) error {
|
||||
for i := range chains {
|
||||
if err := a.AddChain(engine.ContainerTarget(cid), chains[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type frostfsidMock struct {
|
||||
data map[string]*keys.PublicKey
|
||||
}
|
||||
|
||||
func newFrostfsIDMock() *frostfsidMock {
|
||||
return &frostfsidMock{data: map[string]*keys.PublicKey{}}
|
||||
}
|
||||
|
||||
func (f *frostfsidMock) GetUserAddress(account, user string) (string, error) {
|
||||
res, ok := f.data[account+user]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("not found")
|
||||
}
|
||||
|
||||
return res.Address(), nil
|
||||
}
|
||||
|
||||
func (f *frostfsidMock) GetUserKey(account, user string) (string, error) {
|
||||
res, ok := f.data[account+user]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("not found")
|
||||
}
|
||||
|
||||
return hex.EncodeToString(res.Bytes()), nil
|
||||
}
|
||||
|
||||
func createTestBucket(hc *handlerContext, bktName string) *data.BucketInfo {
|
||||
info := createBucket(hc, bktName)
|
||||
return info.BktInfo
|
||||
_, err := hc.MockedPool().CreateContainer(hc.Context(), layer.PrmContainerCreate{
|
||||
Creator: hc.owner,
|
||||
Name: bktName,
|
||||
})
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
bktInfo, err := hc.Layer().GetBucketInfo(hc.Context(), bktName)
|
||||
require.NoError(hc.t, err)
|
||||
return bktInfo
|
||||
}
|
||||
|
||||
func createTestBucketWithLock(hc *handlerContext, bktName string, conf *data.ObjectLockConfiguration) *data.BucketInfo {
|
||||
res, err := hc.MockedPool().CreateContainer(hc.Context(), layer.PrmContainerCreate{
|
||||
cnrID, err := hc.MockedPool().CreateContainer(hc.Context(), layer.PrmContainerCreate{
|
||||
Creator: hc.owner,
|
||||
Name: bktName,
|
||||
AdditionalAttributes: [][2]string{{layer.AttributeLockEnabled, "true"}},
|
||||
|
@ -352,22 +129,17 @@ func createTestBucketWithLock(hc *handlerContext, bktName string, conf *data.Obj
|
|||
var ownerID user.ID
|
||||
|
||||
bktInfo := &data.BucketInfo{
|
||||
CID: res.ContainerID,
|
||||
Name: bktName,
|
||||
ObjectLockEnabled: true,
|
||||
Owner: ownerID,
|
||||
HomomorphicHashDisabled: res.HomomorphicHashDisabled,
|
||||
CID: cnrID,
|
||||
Name: bktName,
|
||||
ObjectLockEnabled: true,
|
||||
Owner: ownerID,
|
||||
}
|
||||
|
||||
key, err := keys.NewPrivateKey()
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
sp := &layer.PutSettingsParams{
|
||||
BktInfo: bktInfo,
|
||||
Settings: &data.BucketSettings{
|
||||
Versioning: data.VersioningEnabled,
|
||||
LockConfiguration: conf,
|
||||
OwnerKey: key.PublicKey(),
|
||||
},
|
||||
}
|
||||
|
||||
|
@ -377,7 +149,7 @@ func createTestBucketWithLock(hc *handlerContext, bktName string, conf *data.Obj
|
|||
return bktInfo
|
||||
}
|
||||
|
||||
func createTestObject(hc *handlerContext, bktInfo *data.BucketInfo, objName string, encryption encryption.Params) *data.ObjectInfo {
|
||||
func createTestObject(hc *handlerContext, bktInfo *data.BucketInfo, objName string) *data.ObjectInfo {
|
||||
content := make([]byte, 1024)
|
||||
_, err := rand.Read(content)
|
||||
require.NoError(hc.t, err)
|
||||
|
@ -387,12 +159,11 @@ func createTestObject(hc *handlerContext, bktInfo *data.BucketInfo, objName stri
|
|||
}
|
||||
|
||||
extObjInfo, err := hc.Layer().PutObject(hc.Context(), &layer.PutObjectParams{
|
||||
BktInfo: bktInfo,
|
||||
Object: objName,
|
||||
Size: uint64(len(content)),
|
||||
Reader: bytes.NewReader(content),
|
||||
Header: header,
|
||||
Encryption: encryption,
|
||||
BktInfo: bktInfo,
|
||||
Object: objName,
|
||||
Size: int64(len(content)),
|
||||
Reader: bytes.NewReader(content),
|
||||
Header: header,
|
||||
})
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
|
@ -415,8 +186,8 @@ func prepareTestRequestWithQuery(hc *handlerContext, bktName, objName string, qu
|
|||
r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(body))
|
||||
r.URL.RawQuery = query.Encode()
|
||||
|
||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
||||
r = r.WithContext(middleware.SetReqInfo(hc.Context(), reqInfo))
|
||||
reqInfo := api.NewReqInfo(w, r, api.ObjectRequest{Bucket: bktName, Object: objName})
|
||||
r = r.WithContext(api.SetReqInfo(hc.Context(), reqInfo))
|
||||
|
||||
return w, r
|
||||
}
|
||||
|
@ -425,8 +196,8 @@ func prepareTestPayloadRequest(hc *handlerContext, bktName, objName string, payl
|
|||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodPut, defaultURL, payload)
|
||||
|
||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
||||
r = r.WithContext(middleware.SetReqInfo(hc.Context(), reqInfo))
|
||||
reqInfo := api.NewReqInfo(w, r, api.ObjectRequest{Bucket: bktName, Object: objName})
|
||||
r = r.WithContext(api.SetReqInfo(hc.Context(), reqInfo))
|
||||
|
||||
return w, r
|
||||
}
|
||||
|
@ -441,16 +212,10 @@ func existInMockedFrostFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo
|
|||
p := &layer.GetObjectParams{
|
||||
BucketInfo: bktInfo,
|
||||
ObjectInfo: objInfo,
|
||||
Writer: io.Discard,
|
||||
}
|
||||
|
||||
objPayload, err := tc.Layer().GetObject(tc.Context(), p)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
_, err = io.ReadAll(objPayload)
|
||||
require.NoError(tc.t, err)
|
||||
return true
|
||||
return tc.Layer().GetObject(tc.Context(), p) == nil
|
||||
}
|
||||
|
||||
func listOIDsFromMockedFrostFS(t *testing.T, tc *handlerContext, bktName string) []oid.ID {
|
||||
|
@ -470,8 +235,6 @@ func assertStatus(t *testing.T, w *httptest.ResponseRecorder, status int) {
|
|||
|
||||
func readResponse(t *testing.T, w *httptest.ResponseRecorder, status int, model interface{}) {
|
||||
assertStatus(t, w, status)
|
||||
if status == http.StatusOK {
|
||||
err := xml.NewDecoder(w.Result().Body).Decode(model)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
err := xml.NewDecoder(w.Result().Body).Decode(model)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
|
|
@ -1,21 +1,20 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"io"
|
||||
"bytes"
|
||||
"net/http"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const sizeToDetectType = 512
|
||||
|
||||
func getRangeToDetectContentType(maxSize uint64) *layer.RangeParams {
|
||||
end := maxSize
|
||||
func getRangeToDetectContentType(maxSize int64) *layer.RangeParams {
|
||||
end := uint64(maxSize)
|
||||
if sizeToDetectType < end {
|
||||
end = sizeToDetectType
|
||||
}
|
||||
|
@ -27,7 +26,7 @@ func getRangeToDetectContentType(maxSize uint64) *layer.RangeParams {
|
|||
}
|
||||
|
||||
func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -65,12 +64,12 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
if err = checkPreconditions(info, conditional, h.cfg.MD5Enabled()); err != nil {
|
||||
if err = checkPreconditions(info, conditional); err != nil {
|
||||
h.logAndSendError(w, "precondition failed", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
t := &data.ObjectVersion{
|
||||
t := &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: info.Name,
|
||||
VersionID: info.VersionID(),
|
||||
|
@ -84,26 +83,18 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
if len(info.ContentType) == 0 {
|
||||
if info.ContentType = layer.MimeByFilePath(info.Name); len(info.ContentType) == 0 {
|
||||
buffer := bytes.NewBuffer(make([]byte, 0, sizeToDetectType))
|
||||
getParams := &layer.GetObjectParams{
|
||||
ObjectInfo: info,
|
||||
Versioned: p.Versioned(),
|
||||
Writer: buffer,
|
||||
Range: getRangeToDetectContentType(info.Size),
|
||||
BucketInfo: bktInfo,
|
||||
}
|
||||
|
||||
objPayload, err := h.obj.GetObject(r.Context(), getParams)
|
||||
if err != nil {
|
||||
if err = h.obj.GetObject(r.Context(), getParams); err != nil {
|
||||
h.logAndSendError(w, "could not get object", reqInfo, err, zap.Stringer("oid", info.ID))
|
||||
return
|
||||
}
|
||||
|
||||
buffer, err := io.ReadAll(objPayload)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not partly read payload to detect content type", reqInfo, err, zap.Stringer("oid", info.ID))
|
||||
return
|
||||
}
|
||||
|
||||
info.ContentType = http.DetectContentType(buffer)
|
||||
info.ContentType = http.DetectContentType(buffer.Bytes())
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -118,12 +109,12 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
writeHeaders(w.Header(), r.Header, extendedInfo, len(tagSet), bktSettings.Unversioned(), h.cfg.MD5Enabled())
|
||||
writeHeaders(w.Header(), r.Header, extendedInfo, len(tagSet), bktSettings.Unversioned())
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -131,22 +122,18 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
w.Header().Set(api.OwnerID, bktInfo.Owner.EncodeToString())
|
||||
w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString())
|
||||
w.Header().Set(api.AmzBucketRegion, bktInfo.LocationConstraint)
|
||||
|
||||
if isAvailableToResolve(bktInfo.Zone, h.cfg.ResolveZoneList(), h.cfg.IsResolveListAllow()) {
|
||||
if isAvailableToResolve(bktInfo.Zone, h.cfg.ResolveZoneList, h.cfg.IsResolveListAllow) {
|
||||
w.Header().Set(api.ContainerName, bktInfo.Name)
|
||||
w.Header().Set(api.ContainerZone, bktInfo.Zone)
|
||||
}
|
||||
|
||||
if err = middleware.WriteResponse(w, http.StatusOK, nil, middleware.MimeNone); err != nil {
|
||||
h.logAndSendError(w, "write response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
api.WriteResponse(w, http.StatusOK, nil, api.MimeNone)
|
||||
}
|
||||
|
||||
func (h *handler) setLockingHeaders(bktInfo *data.BucketInfo, lockInfo data.LockInfo, header http.Header) error {
|
||||
func (h *handler) setLockingHeaders(bktInfo *data.BucketInfo, lockInfo *data.LockInfo, header http.Header) error {
|
||||
if !bktInfo.ObjectLockEnabled {
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -1,18 +1,15 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
@ -27,14 +24,10 @@ func TestConditionalHead(t *testing.T) {
|
|||
tc.Handler().HeadObjectHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
etag := w.Result().Header.Get(api.ETag)
|
||||
etagQuoted := "\"" + etag + "\""
|
||||
|
||||
headers := map[string]string{api.IfMatch: etag}
|
||||
headObject(t, tc, bktName, objName, headers, http.StatusOK)
|
||||
|
||||
headers = map[string]string{api.IfMatch: etagQuoted}
|
||||
headObject(t, tc, bktName, objName, headers, http.StatusOK)
|
||||
|
||||
headers = map[string]string{api.IfMatch: "etag"}
|
||||
headObject(t, tc, bktName, objName, headers, http.StatusPreconditionFailed)
|
||||
|
||||
|
@ -54,9 +47,6 @@ func TestConditionalHead(t *testing.T) {
|
|||
headers = map[string]string{api.IfNoneMatch: etag}
|
||||
headObject(t, tc, bktName, objName, headers, http.StatusNotModified)
|
||||
|
||||
headers = map[string]string{api.IfNoneMatch: etagQuoted}
|
||||
headObject(t, tc, bktName, objName, headers, http.StatusNotModified)
|
||||
|
||||
headers = map[string]string{api.IfNoneMatch: "etag"}
|
||||
headObject(t, tc, bktName, objName, headers, http.StatusOK)
|
||||
|
||||
|
@ -85,48 +75,17 @@ func headObject(t *testing.T, tc *handlerContext, bktName, objName string, heade
|
|||
}
|
||||
|
||||
func TestInvalidAccessThroughCache(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
tc := prepareHandlerContext(t)
|
||||
bktName, objName := "bucket-for-cache", "obj-for-cache"
|
||||
bktInfo, _ := createBucketAndObject(hc, bktName, objName)
|
||||
setContainerEACL(hc, bktInfo.CID)
|
||||
createBucketAndObject(tc, bktName, objName)
|
||||
|
||||
headObject(t, hc, bktName, objName, nil, http.StatusOK)
|
||||
headObject(t, tc, bktName, objName, nil, http.StatusOK)
|
||||
|
||||
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
||||
hc.Handler().HeadObjectHandler(w, r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: newTestAccessBox(t, nil)})))
|
||||
w, r := prepareTestRequest(tc, bktName, objName, nil)
|
||||
tc.Handler().HeadObjectHandler(w, r.WithContext(context.WithValue(r.Context(), api.BoxData, newTestAccessBox(t, nil))))
|
||||
assertStatus(t, w, http.StatusForbidden)
|
||||
}
|
||||
|
||||
func setContainerEACL(hc *handlerContext, cnrID cid.ID) {
|
||||
table := eacl.NewTable()
|
||||
table.SetCID(cnrID)
|
||||
for _, op := range fullOps {
|
||||
table.AddRecord(getOthersRecord(op, eacl.ActionDeny))
|
||||
}
|
||||
|
||||
err := hc.MockedPool().SetContainerEACL(hc.Context(), *table, nil)
|
||||
require.NoError(hc.t, err)
|
||||
}
|
||||
|
||||
func TestHeadObject(t *testing.T) {
|
||||
hc := prepareHandlerContextWithMinCache(t)
|
||||
bktName, objName := "bucket", "obj"
|
||||
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
|
||||
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
|
||||
checkFound(hc.t, hc, bktName, objName, emptyVersion)
|
||||
|
||||
addr := getAddressOfLastVersion(hc, bktInfo, objName)
|
||||
hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
|
||||
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectNotFound{})
|
||||
|
||||
headObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), s3errors.ErrNoSuchVersion)
|
||||
headObjectAssertS3Error(hc, bktName, objName, emptyVersion, s3errors.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
func TestIsAvailableToResolve(t *testing.T) {
|
||||
list := []string{"container", "s3"}
|
||||
|
||||
|
|
|
@ -3,11 +3,11 @@ package handler
|
|||
import (
|
||||
"net/http"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
)
|
||||
|
||||
func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -15,7 +15,7 @@ func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Reques
|
|||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, LocationResponse{Location: bktInfo.LocationConstraint}); err != nil {
|
||||
if err = api.EncodeToResponse(w, LocationResponse{Location: bktInfo.LocationConstraint}); err != nil {
|
||||
h.logAndSendError(w, "couldn't encode bucket location response", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,8 +4,8 @@ import (
|
|||
"net/http"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
||||
)
|
||||
|
||||
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||
|
@ -15,7 +15,7 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
|||
var (
|
||||
own user.ID
|
||||
res *ListBucketsResponse
|
||||
reqInfo = middleware.GetReqInfo(r.Context())
|
||||
reqInfo = api.GetReqInfo(r.Context())
|
||||
)
|
||||
|
||||
list, err := h.obj.ListBuckets(r.Context())
|
||||
|
@ -42,7 +42,7 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
|||
})
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, res); err != nil {
|
||||
if err = api.EncodeToResponse(w, res); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,16 +2,16 @@ package handler
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -26,7 +26,7 @@ const (
|
|||
)
|
||||
|
||||
func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -41,7 +41,7 @@ func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt
|
|||
}
|
||||
|
||||
lockingConf := &data.ObjectLockConfiguration{}
|
||||
if err = h.cfg.NewXMLDecoder(r.Body).Decode(lockingConf); err != nil {
|
||||
if err = xml.NewDecoder(r.Body).Decode(lockingConf); err != nil {
|
||||
h.logAndSendError(w, "couldn't parse locking configuration", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
@ -73,7 +73,7 @@ func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt
|
|||
}
|
||||
|
||||
func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -100,13 +100,13 @@ func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt
|
|||
settings.LockConfiguration.ObjectLockEnabled = enabledValue
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, settings.LockConfiguration); err != nil {
|
||||
if err = api.EncodeToResponse(w, settings.LockConfiguration); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -121,7 +121,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
|
|||
}
|
||||
|
||||
legalHold := &data.LegalHold{}
|
||||
if err = h.cfg.NewXMLDecoder(r.Body).Decode(legalHold); err != nil {
|
||||
if err = xml.NewDecoder(r.Body).Decode(legalHold); err != nil {
|
||||
h.logAndSendError(w, "couldn't parse legal hold configuration", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
@ -133,7 +133,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
|
|||
}
|
||||
|
||||
p := &layer.PutLockInfoParams{
|
||||
ObjVersion: &data.ObjectVersion{
|
||||
ObjVersion: &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||
|
@ -143,12 +143,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
|
|||
Enabled: legalHold.Status == legalHoldOn,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||
return
|
||||
CopiesNumber: h.cfg.CopiesNumber,
|
||||
}
|
||||
|
||||
if err = h.obj.PutLockInfo(r.Context(), p); err != nil {
|
||||
|
@ -158,7 +153,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
|
|||
}
|
||||
|
||||
func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -172,7 +167,7 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
|
|||
return
|
||||
}
|
||||
|
||||
p := &data.ObjectVersion{
|
||||
p := &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||
|
@ -189,13 +184,13 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
|
|||
legalHold.Status = legalHoldOn
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, legalHold); err != nil {
|
||||
if err = api.EncodeToResponse(w, legalHold); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -209,7 +204,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
|
|||
}
|
||||
|
||||
retention := &data.Retention{}
|
||||
if err = h.cfg.NewXMLDecoder(r.Body).Decode(retention); err != nil {
|
||||
if err = xml.NewDecoder(r.Body).Decode(retention); err != nil {
|
||||
h.logAndSendError(w, "couldn't parse object retention", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
@ -221,18 +216,13 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
|
|||
}
|
||||
|
||||
p := &layer.PutLockInfoParams{
|
||||
ObjVersion: &data.ObjectVersion{
|
||||
ObjVersion: &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||
},
|
||||
NewLock: lock,
|
||||
}
|
||||
|
||||
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||
return
|
||||
NewLock: lock,
|
||||
CopiesNumber: h.cfg.CopiesNumber,
|
||||
}
|
||||
|
||||
if err = h.obj.PutLockInfo(r.Context(), p); err != nil {
|
||||
|
@ -242,7 +232,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
|
|||
}
|
||||
|
||||
func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -256,7 +246,7 @@ func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
|
|||
return
|
||||
}
|
||||
|
||||
p := &data.ObjectVersion{
|
||||
p := &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||
|
@ -281,7 +271,7 @@ func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
|
|||
retention.Mode = complianceMode
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, retention); err != nil {
|
||||
if err = api.EncodeToResponse(w, retention); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -10,11 +10,9 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -315,7 +313,7 @@ func TestPutBucketLockConfigurationHandler(t *testing.T) {
|
|||
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(body))
|
||||
r = r.WithContext(middleware.SetReqInfo(r.Context(), middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: tc.bucket}, "")))
|
||||
r = r.WithContext(api.SetReqInfo(r.Context(), api.NewReqInfo(w, r, api.ObjectRequest{Bucket: tc.bucket})))
|
||||
|
||||
hc.Handler().PutBucketObjectLockConfigHandler(w, r)
|
||||
|
||||
|
@ -388,7 +386,7 @@ func TestGetBucketLockConfigurationHandler(t *testing.T) {
|
|||
t.Run(tc.name, func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(nil))
|
||||
r = r.WithContext(middleware.SetReqInfo(r.Context(), middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: tc.bucket}, "")))
|
||||
r = r.WithContext(api.SetReqInfo(r.Context(), api.NewReqInfo(w, r, api.ObjectRequest{Bucket: tc.bucket})))
|
||||
|
||||
hc.Handler().GetBucketObjectLockConfigHandler(w, r)
|
||||
|
||||
|
@ -408,7 +406,7 @@ func TestGetBucketLockConfigurationHandler(t *testing.T) {
|
|||
}
|
||||
|
||||
func assertS3Error(t *testing.T, w *httptest.ResponseRecorder, expectedError apiErrors.Error) {
|
||||
actualErrorResponse := &middleware.ErrorResponse{}
|
||||
actualErrorResponse := &api.ErrorResponse{}
|
||||
err := xml.NewDecoder(w.Result().Body).Decode(actualErrorResponse)
|
||||
require.NoError(t, err)
|
||||
|
||||
|
@ -427,7 +425,7 @@ func TestObjectLegalHold(t *testing.T) {
|
|||
bktInfo := createTestBucketWithLock(hc, bktName, nil)
|
||||
|
||||
objName := "obj-for-legal-hold"
|
||||
createTestObject(hc, bktInfo, objName, encryption.Params{})
|
||||
createTestObject(hc, bktInfo, objName)
|
||||
|
||||
getObjectLegalHold(hc, bktName, objName, legalHoldOff)
|
||||
|
||||
|
@ -471,7 +469,7 @@ func TestObjectRetention(t *testing.T) {
|
|||
bktInfo := createTestBucketWithLock(hc, bktName, nil)
|
||||
|
||||
objName := "obj-for-retention"
|
||||
createTestObject(hc, bktInfo, objName, encryption.Params{})
|
||||
createTestObject(hc, bktInfo, objName)
|
||||
|
||||
getObjectRetention(hc, bktName, objName, nil, apiErrors.ErrNoSuchKey)
|
||||
|
||||
|
@ -537,7 +535,7 @@ func TestPutObjectWithLock(t *testing.T) {
|
|||
createTestBucketWithLock(hc, bktName, lockConfig)
|
||||
|
||||
objDefault := "obj-default-retention"
|
||||
putObject(hc, bktName, objDefault)
|
||||
putObject(t, hc, bktName, objDefault)
|
||||
|
||||
getObjectRetentionApproximate(hc, bktName, objDefault, governanceMode, time.Now().Add(24*time.Hour))
|
||||
getObjectLegalHold(hc, bktName, objDefault, legalHoldOff)
|
||||
|
@ -588,7 +586,7 @@ func TestPutLockErrors(t *testing.T) {
|
|||
headers[api.AmzObjectLockRetainUntilDate] = "dummy"
|
||||
putObjectWithLockFailed(t, hc, bktName, objName, headers, apiErrors.ErrInvalidRetentionDate)
|
||||
|
||||
putObject(hc, bktName, objName)
|
||||
putObject(t, hc, bktName, objName)
|
||||
|
||||
retention := &data.Retention{Mode: governanceMode}
|
||||
putObjectRetentionFailed(t, hc, bktName, objName, retention, apiErrors.ErrMalformedXML)
|
||||
|
|
|
@ -2,18 +2,16 @@ package handler
|
|||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"github.com/google/uuid"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -60,7 +58,7 @@ type (
|
|||
Owner Owner `xml:"Owner"`
|
||||
Parts []*layer.Part `xml:"Part"`
|
||||
PartNumberMarker int `xml:"PartNumberMarker,omitempty"`
|
||||
StorageClass string `xml:"StorageClass"`
|
||||
StorageClass string `xml:"StorageClass,omitempty"`
|
||||
UploadID string `xml:"UploadId"`
|
||||
}
|
||||
|
||||
|
@ -69,7 +67,7 @@ type (
|
|||
Initiator Initiator `xml:"Initiator"`
|
||||
Key string `xml:"Key"`
|
||||
Owner Owner `xml:"Owner"`
|
||||
StorageClass string `xml:"StorageClass"`
|
||||
StorageClass string `xml:"StorageClass,omitempty"`
|
||||
UploadID string `xml:"UploadId"`
|
||||
}
|
||||
|
||||
|
@ -92,20 +90,10 @@ type (
|
|||
const (
|
||||
uploadIDHeaderName = "uploadId"
|
||||
partNumberHeaderName = "partNumber"
|
||||
|
||||
prefixQueryName = "prefix"
|
||||
delimiterQueryName = "delimiter"
|
||||
maxUploadsQueryName = "max-uploads"
|
||||
encodingTypeQueryName = "encoding-type"
|
||||
keyMarkerQueryName = "key-marker"
|
||||
uploadIDMarkerQueryName = "upload-id-marker"
|
||||
)
|
||||
|
||||
func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
uploadID := uuid.New()
|
||||
cannedACLStatus := aclHeadersStatus(r)
|
||||
additional := []zap.Field{zap.String("uploadID", uploadID.String())}
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -113,16 +101,10 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
|||
return
|
||||
}
|
||||
|
||||
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
apeEnabled := bktInfo.APEEnabled || settings.CannedACL != ""
|
||||
if apeEnabled && cannedACLStatus == aclStatusYes {
|
||||
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
||||
return
|
||||
uploadID := uuid.New()
|
||||
additional := []zap.Field{
|
||||
zap.String("uploadID", uploadID.String()),
|
||||
zap.String("Key", reqInfo.ObjectName),
|
||||
}
|
||||
|
||||
p := &layer.CreateMultipartParams{
|
||||
|
@ -134,15 +116,14 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
|||
Data: &layer.UploadData{},
|
||||
}
|
||||
|
||||
needUpdateEACLTable := !(apeEnabled || cannedACLStatus == aclStatusNo)
|
||||
if needUpdateEACLTable {
|
||||
if containsACLHeaders(r) {
|
||||
key, err := h.bearerTokenIssuerKey(r.Context())
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't get gate key", reqInfo, err, additional...)
|
||||
h.logAndSendError(w, "couldn't get gate key", reqInfo, err)
|
||||
return
|
||||
}
|
||||
if _, err = parseACLHeaders(r.Header, key); err != nil {
|
||||
h.logAndSendError(w, "could not parse acl", reqInfo, err, additional...)
|
||||
h.logAndSendError(w, "could not parse acl", reqInfo, err)
|
||||
return
|
||||
}
|
||||
p.Data.ACLHeaders = formACLHeadersForMultipart(r.Header)
|
||||
|
@ -158,7 +139,7 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
|||
|
||||
p.Info.Encryption, err = formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err, additional...)
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -166,13 +147,10 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
|||
if contentType := r.Header.Get(api.ContentType); len(contentType) > 0 {
|
||||
p.Header[api.ContentType] = contentType
|
||||
}
|
||||
if contentLanguage := r.Header.Get(api.ContentLanguage); len(contentLanguage) > 0 {
|
||||
p.Header[api.ContentLanguage] = contentLanguage
|
||||
}
|
||||
|
||||
p.CopiesNumbers, err = h.pickCopiesNumbers(p.Header, reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||
p.CopiesNumber, err = getCopiesNumberOrDefault(p.Header, h.cfg.CopiesNumber)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid copies number", reqInfo, err, additional...)
|
||||
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -191,7 +169,7 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
|||
UploadID: uploadID.String(),
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, resp); err != nil {
|
||||
if err = api.EncodeToResponse(w, resp); err != nil {
|
||||
h.logAndSendError(w, "could not encode InitiateMultipartUploadResponse to response", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
|
@ -217,7 +195,7 @@ func formACLHeadersForMultipart(header http.Header) map[string]string {
|
|||
}
|
||||
|
||||
func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -228,43 +206,29 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
|||
var (
|
||||
queryValues = r.URL.Query()
|
||||
uploadID = queryValues.Get(uploadIDHeaderName)
|
||||
partNumStr = queryValues.Get(partNumberHeaderName)
|
||||
additional = []zap.Field{zap.String("uploadID", uploadID), zap.String("partNumber", partNumStr)}
|
||||
additional = []zap.Field{zap.String("uploadID", uploadID), zap.String("Key", reqInfo.ObjectName)}
|
||||
)
|
||||
|
||||
partNumber, err := strconv.Atoi(partNumStr)
|
||||
partNumber, err := strconv.Atoi(queryValues.Get(partNumberHeaderName))
|
||||
if err != nil || partNumber < layer.UploadMinPartNumber || partNumber > layer.UploadMaxPartNumber {
|
||||
h.logAndSendError(w, "invalid part number", reqInfo, errors.GetAPIError(errors.ErrInvalidPartNumber), additional...)
|
||||
h.logAndSendError(w, "invalid part number", reqInfo, errors.GetAPIError(errors.ErrInvalidPartNumber))
|
||||
return
|
||||
}
|
||||
|
||||
body, err := h.getBodyReader(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "failed to get body reader", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
|
||||
var size uint64
|
||||
if r.ContentLength > 0 {
|
||||
size = uint64(r.ContentLength)
|
||||
}
|
||||
|
||||
p := &layer.UploadPartParams{
|
||||
Info: &layer.UploadInfoParams{
|
||||
UploadID: uploadID,
|
||||
Bkt: bktInfo,
|
||||
Key: reqInfo.ObjectName,
|
||||
},
|
||||
PartNumber: partNumber,
|
||||
Size: size,
|
||||
Reader: body,
|
||||
ContentMD5: r.Header.Get(api.ContentMD5),
|
||||
ContentSHA256Hash: r.Header.Get(api.AmzContentSha256),
|
||||
PartNumber: partNumber,
|
||||
Size: r.ContentLength,
|
||||
Reader: r.Body,
|
||||
}
|
||||
|
||||
p.Info.Encryption, err = formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err, additional...)
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -278,27 +242,22 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
|||
addSSECHeaders(w.Header(), r.Header)
|
||||
}
|
||||
|
||||
w.Header().Set(api.ETag, data.Quote(hash))
|
||||
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
|
||||
h.logAndSendError(w, "write response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set(api.ETag, hash)
|
||||
api.WriteSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
versionID string
|
||||
ctx = r.Context()
|
||||
reqInfo = middleware.GetReqInfo(ctx)
|
||||
reqInfo = api.GetReqInfo(r.Context())
|
||||
queryValues = reqInfo.URL.Query()
|
||||
uploadID = queryValues.Get(uploadIDHeaderName)
|
||||
partNumStr = queryValues.Get(partNumberHeaderName)
|
||||
additional = []zap.Field{zap.String("uploadID", uploadID), zap.String("partNumber", partNumStr)}
|
||||
additional = []zap.Field{zap.String("uploadID", uploadID), zap.String("Key", reqInfo.ObjectName)}
|
||||
)
|
||||
|
||||
partNumber, err := strconv.Atoi(partNumStr)
|
||||
partNumber, err := strconv.Atoi(queryValues.Get(partNumberHeaderName))
|
||||
if err != nil || partNumber < layer.UploadMinPartNumber || partNumber > layer.UploadMaxPartNumber {
|
||||
h.logAndSendError(w, "invalid part number", reqInfo, errors.GetAPIError(errors.ErrInvalidPartNumber), additional...)
|
||||
h.logAndSendError(w, "invalid part number", reqInfo, errors.GetAPIError(errors.ErrInvalidPartNumber))
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -309,7 +268,7 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
srcBucket, srcObject, err := path2BucketObject(src)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid source copy", reqInfo, err, additional...)
|
||||
h.logAndSendError(w, "invalid source copy", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -322,23 +281,21 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
srcBktInfo, err := h.getBucketAndCheckOwner(r, srcBucket, api.AmzSourceExpectedBucketOwner)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get source bucket info", reqInfo, err, additional...)
|
||||
h.logAndSendError(w, "could not get source bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get target bucket info", reqInfo, err, additional...)
|
||||
h.logAndSendError(w, "could not get target bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
headPrm := &layer.HeadObjectParams{
|
||||
srcInfo, err := h.obj.GetObjectInfo(r.Context(), &layer.HeadObjectParams{
|
||||
BktInfo: srcBktInfo,
|
||||
Object: srcObject,
|
||||
VersionID: versionID,
|
||||
}
|
||||
|
||||
srcInfo, err := h.obj.GetObjectInfo(ctx, headPrm)
|
||||
})
|
||||
if err != nil {
|
||||
if errors.IsS3Error(err, errors.ErrNoSuchKey) && versionID != "" {
|
||||
h.logAndSendError(w, "could not head source object version", reqInfo,
|
||||
|
@ -356,65 +313,57 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
if err = checkPreconditions(srcInfo, args.Conditional, h.cfg.MD5Enabled()); err != nil {
|
||||
if err = checkPreconditions(srcInfo, args.Conditional); err != nil {
|
||||
h.logAndSendError(w, "precondition failed", reqInfo, errors.GetAPIError(errors.ErrPreconditionFailed),
|
||||
additional...)
|
||||
return
|
||||
}
|
||||
|
||||
srcEncryptionParams, err := formCopySourceEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = srcEncryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(srcInfo.Headers)); err != nil {
|
||||
h.logAndSendError(w, "encryption doesn't match object", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrBadRequest), err), additional...)
|
||||
return
|
||||
}
|
||||
|
||||
p := &layer.UploadCopyParams{
|
||||
Versioned: headPrm.Versioned(),
|
||||
Info: &layer.UploadInfoParams{
|
||||
UploadID: uploadID,
|
||||
Bkt: bktInfo,
|
||||
Key: reqInfo.ObjectName,
|
||||
},
|
||||
SrcObjInfo: srcInfo,
|
||||
SrcBktInfo: srcBktInfo,
|
||||
SrcEncryption: srcEncryptionParams,
|
||||
PartNumber: partNumber,
|
||||
Range: srcRange,
|
||||
SrcObjInfo: srcInfo,
|
||||
SrcBktInfo: srcBktInfo,
|
||||
PartNumber: partNumber,
|
||||
Range: srcRange,
|
||||
}
|
||||
|
||||
p.Info.Encryption, err = formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err, additional...)
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
info, err := h.obj.UploadPartCopy(ctx, p)
|
||||
if err = p.Info.Encryption.MatchObjectEncryption(layer.FormEncryptionInfo(srcInfo.Headers)); err != nil {
|
||||
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
info, err := h.obj.UploadPartCopy(r.Context(), p)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not upload part copy", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
|
||||
response := UploadPartCopyResponse{
|
||||
ETag: info.HashSum,
|
||||
LastModified: info.Created.UTC().Format(time.RFC3339),
|
||||
ETag: data.Quote(info.ETag(h.cfg.MD5Enabled())),
|
||||
}
|
||||
|
||||
if p.Info.Encryption.Enabled() {
|
||||
addSSECHeaders(w.Header(), r.Header)
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, response); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err, additional...)
|
||||
if err = api.EncodeToResponse(w, response); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -422,26 +371,22 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
|||
return
|
||||
}
|
||||
|
||||
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
sessionTokenSetEACL *session.Container
|
||||
|
||||
uploadID = r.URL.Query().Get(uploadIDHeaderName)
|
||||
uploadInfo = &layer.UploadInfoParams{
|
||||
UploadID: uploadID,
|
||||
Bkt: bktInfo,
|
||||
Key: reqInfo.ObjectName,
|
||||
}
|
||||
additional = []zap.Field{zap.String("uploadID", uploadID)}
|
||||
additional = []zap.Field{zap.String("uploadID", uploadID), zap.String("Key", reqInfo.ObjectName)}
|
||||
)
|
||||
|
||||
reqBody := new(CompleteMultipartUpload)
|
||||
if err = h.cfg.NewXMLDecoder(r.Body).Decode(reqBody); err != nil {
|
||||
if err = xml.NewDecoder(r.Body).Decode(reqBody); err != nil {
|
||||
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
|
||||
fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()), additional...)
|
||||
errors.GetAPIError(errors.ErrMalformedXML), additional...)
|
||||
return
|
||||
}
|
||||
if len(reqBody.Parts) == 0 {
|
||||
|
@ -454,41 +399,16 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
|||
Parts: reqBody.Parts,
|
||||
}
|
||||
|
||||
// Start complete multipart upload which may take some time to fetch object
|
||||
// and re-upload it part by part.
|
||||
objInfo, err := h.completeMultipartUpload(r, c, bktInfo, reqInfo)
|
||||
|
||||
uploadData, extendedObjInfo, err := h.obj.CompleteMultipartUpload(r.Context(), c)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "complete multipart error", reqInfo, err, additional...)
|
||||
h.logAndSendError(w, "could not complete multipart upload", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
|
||||
response := CompleteMultipartUploadResponse{
|
||||
Bucket: objInfo.Bucket,
|
||||
Key: objInfo.Name,
|
||||
ETag: data.Quote(objInfo.ETag(h.cfg.MD5Enabled())),
|
||||
}
|
||||
|
||||
if settings.VersioningEnabled() {
|
||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, response); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err, additional...)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMultipartParams, bktInfo *data.BucketInfo, reqInfo *middleware.ReqInfo) (*data.ObjectInfo, error) {
|
||||
ctx := r.Context()
|
||||
uploadData, extendedObjInfo, err := h.obj.CompleteMultipartUpload(ctx, c)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not complete multipart upload: %w", err)
|
||||
}
|
||||
objInfo := extendedObjInfo.ObjectInfo
|
||||
|
||||
if len(uploadData.TagSet) != 0 {
|
||||
tagPrm := &data.PutObjectTaggingParams{
|
||||
ObjectVersion: &data.ObjectVersion{
|
||||
tagPrm := &layer.PutObjectTaggingParams{
|
||||
ObjectVersion: &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: objInfo.Name,
|
||||
VersionID: objInfo.VersionID(),
|
||||
|
@ -496,23 +416,22 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
|
|||
TagSet: uploadData.TagSet,
|
||||
NodeVersion: extendedObjInfo.NodeVersion,
|
||||
}
|
||||
if _, err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
|
||||
return nil, fmt.Errorf("could not put tagging file of completed multipart upload: %w", err)
|
||||
if _, err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
|
||||
h.logAndSendError(w, "could not put tagging file of completed multipart upload", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if len(uploadData.ACLHeaders) != 0 {
|
||||
sessionTokenSetEACL, err := getSessionTokenSetEACL(ctx)
|
||||
key, err := h.bearerTokenIssuerKey(r.Context())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get eacl token: %w", err)
|
||||
}
|
||||
key, err := h.bearerTokenIssuerKey(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get gate key: %w", err)
|
||||
h.logAndSendError(w, "couldn't get gate key", reqInfo, err)
|
||||
return
|
||||
}
|
||||
acl, err := parseACLHeaders(r.Header, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse acl: %w", err)
|
||||
h.logAndSendError(w, "could not parse acl", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
resInfo := &resourceInfo{
|
||||
|
@ -521,28 +440,47 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
|
|||
}
|
||||
astObject, err := aclToAst(acl, resInfo)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not translate acl of completed multipart upload to ast: %w", err)
|
||||
h.logAndSendError(w, "could not translate acl of completed multipart upload to ast", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
if _, err = h.updateBucketACL(r, astObject, bktInfo, sessionTokenSetEACL); err != nil {
|
||||
return nil, fmt.Errorf("could not update bucket acl while completing multipart upload: %w", err)
|
||||
h.logAndSendError(w, "could not update bucket acl while completing multipart upload", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
s := &SendNotificationParams{
|
||||
Event: EventObjectCreatedCompleteMultipartUpload,
|
||||
NotificationInfo: data.NotificationInfoFromObject(objInfo, h.cfg.MD5Enabled()),
|
||||
NotificationInfo: data.NotificationInfoFromObject(objInfo),
|
||||
BktInfo: bktInfo,
|
||||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
if err = h.sendNotifications(r.Context(), s); err != nil {
|
||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||
}
|
||||
|
||||
return objInfo, nil
|
||||
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||
}
|
||||
|
||||
response := CompleteMultipartUploadResponse{
|
||||
Bucket: objInfo.Bucket,
|
||||
ETag: objInfo.HashSum,
|
||||
Key: objInfo.Name,
|
||||
}
|
||||
|
||||
if bktSettings.VersioningEnabled() {
|
||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
||||
}
|
||||
|
||||
if err = api.EncodeToResponse(w, response); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -551,28 +489,31 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req
|
|||
}
|
||||
|
||||
var (
|
||||
queryValues = reqInfo.URL.Query()
|
||||
maxUploadsStr = queryValues.Get(maxUploadsQueryName)
|
||||
maxUploads = layer.MaxSizeUploadsList
|
||||
queryValues = reqInfo.URL.Query()
|
||||
delimiter = queryValues.Get("delimiter")
|
||||
prefix = queryValues.Get("prefix")
|
||||
maxUploads = layer.MaxSizeUploadsList
|
||||
)
|
||||
|
||||
if maxUploadsStr != "" {
|
||||
val, err := strconv.Atoi(maxUploadsStr)
|
||||
if err != nil || val < 1 || val > 1000 {
|
||||
if queryValues.Get("max-uploads") != "" {
|
||||
val, err := strconv.Atoi(queryValues.Get("max-uploads"))
|
||||
if err != nil || val < 0 {
|
||||
h.logAndSendError(w, "invalid maxUploads", reqInfo, errors.GetAPIError(errors.ErrInvalidMaxUploads))
|
||||
return
|
||||
}
|
||||
maxUploads = val
|
||||
if val < maxUploads {
|
||||
maxUploads = val
|
||||
}
|
||||
}
|
||||
|
||||
p := &layer.ListMultipartUploadsParams{
|
||||
Bkt: bktInfo,
|
||||
Delimiter: queryValues.Get(delimiterQueryName),
|
||||
EncodingType: queryValues.Get(encodingTypeQueryName),
|
||||
KeyMarker: queryValues.Get(keyMarkerQueryName),
|
||||
Delimiter: delimiter,
|
||||
EncodingType: queryValues.Get("encoding-type"),
|
||||
KeyMarker: queryValues.Get("key-marker"),
|
||||
MaxUploads: maxUploads,
|
||||
Prefix: queryValues.Get(prefixQueryName),
|
||||
UploadIDMarker: queryValues.Get(uploadIDMarkerQueryName),
|
||||
Prefix: prefix,
|
||||
UploadIDMarker: queryValues.Get("upload-id-marker"),
|
||||
}
|
||||
|
||||
list, err := h.obj.ListMultipartUploads(r.Context(), p)
|
||||
|
@ -581,13 +522,13 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req
|
|||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, encodeListMultipartUploadsToResponse(list, p)); err != nil {
|
||||
if err = api.EncodeToResponse(w, encodeListMultipartUploadsToResponse(list, p)); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -616,7 +557,7 @@ func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
if queryValues.Get("part-number-marker") != "" {
|
||||
if partNumberMarker, err = strconv.Atoi(queryValues.Get("part-number-marker")); err != nil || partNumberMarker < 0 {
|
||||
if partNumberMarker, err = strconv.Atoi(queryValues.Get("part-number-marker")); err != nil || partNumberMarker <= 0 {
|
||||
h.logAndSendError(w, "invalid PartNumberMarker", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
|
@ -644,13 +585,13 @@ func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, encodeListPartsToResponse(list, p)); err != nil {
|
||||
if err = api.EncodeToResponse(w, encodeListPartsToResponse(list, p)); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -709,8 +650,7 @@ func encodeListMultipartUploadsToResponse(info *layer.ListMultipartUploadsInfo,
|
|||
ID: u.Owner.String(),
|
||||
DisplayName: u.Owner.String(),
|
||||
},
|
||||
UploadID: u.UploadID,
|
||||
StorageClass: api.DefaultStorageClass,
|
||||
UploadID: u.UploadID,
|
||||
}
|
||||
uploads = append(uploads, m)
|
||||
}
|
||||
|
@ -739,6 +679,5 @@ func encodeListPartsToResponse(info *layer.ListPartsInfo, params *layer.ListPart
|
|||
PartNumberMarker: params.PartNumberMarker,
|
||||
UploadID: params.Info.UploadID,
|
||||
Parts: info.Parts,
|
||||
StorageClass: api.DefaultStorageClass,
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,492 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3Errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
partNumberMarkerQuery = "part-number-marker"
|
||||
)
|
||||
|
||||
func TestMultipartUploadInvalidPart(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-to-upload-part", "object-multipart"
|
||||
createTestBucket(hc, bktName)
|
||||
partSize := 8 // less than min part size
|
||||
|
||||
multipartUpload := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
||||
etag1, _ := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 1, partSize)
|
||||
etag2, _ := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 2, partSize)
|
||||
w := completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1, etag2})
|
||||
assertS3Error(hc.t, w, s3Errors.GetAPIError(s3Errors.ErrEntityTooSmall))
|
||||
}
|
||||
|
||||
func TestDeleteMultipartAllParts(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
partSize := layer.UploadMinSize
|
||||
objLen := 6 * partSize
|
||||
|
||||
bktName, bktName2, objName := "bucket", "bucket2", "object"
|
||||
|
||||
// unversioned bucket
|
||||
createTestBucket(hc, bktName)
|
||||
multipartUpload(hc, bktName, objName, nil, objLen, partSize)
|
||||
deleteObject(t, hc, bktName, objName, emptyVersion)
|
||||
require.Empty(t, hc.tp.Objects())
|
||||
|
||||
// encrypted multipart
|
||||
multipartUploadEncrypted(hc, bktName, objName, nil, objLen, partSize)
|
||||
deleteObject(t, hc, bktName, objName, emptyVersion)
|
||||
require.Empty(t, hc.tp.Objects())
|
||||
|
||||
// versions bucket
|
||||
createTestBucket(hc, bktName2)
|
||||
putBucketVersioning(t, hc, bktName2, true)
|
||||
multipartUpload(hc, bktName2, objName, nil, objLen, partSize)
|
||||
_, hdr := getObject(hc, bktName2, objName)
|
||||
versionID := hdr.Get("X-Amz-Version-Id")
|
||||
deleteObject(t, hc, bktName2, objName, emptyVersion)
|
||||
deleteObject(t, hc, bktName2, objName, versionID)
|
||||
require.Empty(t, hc.tp.Objects())
|
||||
}
|
||||
|
||||
func TestMultipartReUploadPart(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-to-upload-part", "object-multipart"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
partSizeLast := 8 // less than min part size
|
||||
partSizeFirst := 5 * 1024 * 1024
|
||||
|
||||
uploadInfo := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
||||
etag1, _ := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSizeLast)
|
||||
etag2, _ := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 2, partSizeFirst)
|
||||
|
||||
list := listParts(hc, bktName, objName, uploadInfo.UploadID, "0", http.StatusOK)
|
||||
require.Len(t, list.Parts, 2)
|
||||
require.Equal(t, etag1, list.Parts[0].ETag)
|
||||
require.Equal(t, etag2, list.Parts[1].ETag)
|
||||
|
||||
w := completeMultipartUploadBase(hc, bktName, objName, uploadInfo.UploadID, []string{etag1, etag2})
|
||||
assertS3Error(hc.t, w, s3Errors.GetAPIError(s3Errors.ErrEntityTooSmall))
|
||||
|
||||
etag1, data1 := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSizeFirst)
|
||||
etag2, data2 := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 2, partSizeLast)
|
||||
|
||||
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "0", http.StatusOK)
|
||||
require.Len(t, list.Parts, 2)
|
||||
require.Equal(t, etag1, list.Parts[0].ETag)
|
||||
require.Equal(t, etag2, list.Parts[1].ETag)
|
||||
|
||||
innerUploadInfo, err := hc.tree.GetMultipartUpload(hc.context, bktInfo, objName, uploadInfo.UploadID)
|
||||
require.NoError(t, err)
|
||||
treeParts, err := hc.tree.GetParts(hc.Context(), bktInfo, innerUploadInfo.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, treeParts, len(list.Parts))
|
||||
|
||||
w = completeMultipartUploadBase(hc, bktName, objName, uploadInfo.UploadID, []string{etag1, etag2})
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
|
||||
data, _ := getObject(hc, bktName, objName)
|
||||
equalDataSlices(t, append(data1, data2...), data)
|
||||
}
|
||||
|
||||
func TestListMultipartUploads(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-to-list-uploads"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objName1 := "/my/object/name"
|
||||
uploadInfo1 := createMultipartUpload(hc, bktName, objName1, map[string]string{})
|
||||
objName2 := "/my/object2"
|
||||
uploadInfo2 := createMultipartUpload(hc, bktName, objName2, map[string]string{})
|
||||
objName3 := "/zzz/object/name3"
|
||||
uploadInfo3 := createMultipartUpload(hc, bktName, objName3, map[string]string{})
|
||||
|
||||
t.Run("check upload key", func(t *testing.T) {
|
||||
listUploads := listAllMultipartUploads(hc, bktName)
|
||||
require.Len(t, listUploads.Uploads, 3)
|
||||
for i, upload := range []*InitiateMultipartUploadResponse{uploadInfo1, uploadInfo2, uploadInfo3} {
|
||||
require.Equal(t, upload.UploadID, listUploads.Uploads[i].UploadID)
|
||||
require.Equal(t, upload.Key, listUploads.Uploads[i].Key)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("check max uploads", func(t *testing.T) {
|
||||
listUploads := listMultipartUploadsBase(hc, bktName, "", "", "", "", 2)
|
||||
require.Len(t, listUploads.Uploads, 2)
|
||||
require.Equal(t, uploadInfo1.UploadID, listUploads.Uploads[0].UploadID)
|
||||
require.Equal(t, uploadInfo2.UploadID, listUploads.Uploads[1].UploadID)
|
||||
})
|
||||
|
||||
t.Run("check prefix", func(t *testing.T) {
|
||||
listUploads := listMultipartUploadsBase(hc, bktName, "/my", "", "", "", -1)
|
||||
require.Len(t, listUploads.Uploads, 2)
|
||||
require.Equal(t, uploadInfo1.UploadID, listUploads.Uploads[0].UploadID)
|
||||
require.Equal(t, uploadInfo2.UploadID, listUploads.Uploads[1].UploadID)
|
||||
})
|
||||
|
||||
t.Run("check markers", func(t *testing.T) {
|
||||
t.Run("check only key-marker", func(t *testing.T) {
|
||||
listUploads := listMultipartUploadsBase(hc, bktName, "", "", "", objName2, -1)
|
||||
require.Len(t, listUploads.Uploads, 1)
|
||||
// If upload-id-marker is not specified, only the keys lexicographically greater than the specified key-marker will be included in the list.
|
||||
require.Equal(t, uploadInfo3.UploadID, listUploads.Uploads[0].UploadID)
|
||||
})
|
||||
|
||||
t.Run("check only upload-id-marker", func(t *testing.T) {
|
||||
uploadIDMarker := uploadInfo1.UploadID
|
||||
if uploadIDMarker > uploadInfo2.UploadID {
|
||||
uploadIDMarker = uploadInfo2.UploadID
|
||||
}
|
||||
listUploads := listMultipartUploadsBase(hc, bktName, "", "", uploadIDMarker, "", -1)
|
||||
// If key-marker is not specified, the upload-id-marker parameter is ignored.
|
||||
require.Len(t, listUploads.Uploads, 3)
|
||||
})
|
||||
|
||||
t.Run("check key-marker along with upload-id-marker", func(t *testing.T) {
|
||||
uploadIDMarker := "00000000-0000-0000-0000-000000000000"
|
||||
|
||||
listUploads := listMultipartUploadsBase(hc, bktName, "", "", uploadIDMarker, objName3, -1)
|
||||
require.Len(t, listUploads.Uploads, 1)
|
||||
// If upload-id-marker is specified, any multipart uploads for a key equal to the key-marker might also be included,
|
||||
// provided those multipart uploads have upload IDs lexicographically greater than the specified upload-id-marker.
|
||||
require.Equal(t, uploadInfo3.UploadID, listUploads.Uploads[0].UploadID)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestMultipartUploadSize(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-test-multipart-size", "object-multipart"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
partSize := layer.UploadMinSize
|
||||
objLen := 2 * partSize
|
||||
headers := map[string]string{}
|
||||
|
||||
data := multipartUpload(hc, bktName, objName, headers, objLen, partSize)
|
||||
require.Equal(t, objLen, len(data))
|
||||
|
||||
t.Run("check correct size in list v1", func(t *testing.T) {
|
||||
listV1 := listObjectsV1(hc, bktName, "", "", "", -1)
|
||||
require.Len(t, listV1.Contents, 1)
|
||||
require.Equal(t, objLen, int(listV1.Contents[0].Size))
|
||||
require.Equal(t, objName, listV1.Contents[0].Key)
|
||||
})
|
||||
|
||||
t.Run("check correct size in list v2", func(t *testing.T) {
|
||||
listV2 := listObjectsV2(hc, bktName, "", "", "", "", -1)
|
||||
require.Len(t, listV2.Contents, 1)
|
||||
require.Equal(t, objLen, int(listV2.Contents[0].Size))
|
||||
require.Equal(t, objName, listV2.Contents[0].Key)
|
||||
})
|
||||
|
||||
t.Run("check correct get", func(t *testing.T) {
|
||||
_, hdr := getObject(hc, bktName, objName)
|
||||
require.Equal(t, strconv.Itoa(objLen), hdr.Get(api.ContentLength))
|
||||
|
||||
part := getObjectRange(t, hc, bktName, objName, partSize, objLen-1)
|
||||
equalDataSlices(t, data[partSize:], part)
|
||||
})
|
||||
|
||||
t.Run("check correct size when part copy", func(_ *testing.T) {
|
||||
objName2 := "obj2"
|
||||
uploadInfo := createMultipartUpload(hc, bktName, objName2, headers)
|
||||
sourceCopy := bktName + "/" + objName
|
||||
uploadPartCopy(hc, bktName, objName2, uploadInfo.UploadID, 1, sourceCopy, 0, 0)
|
||||
uploadPartCopy(hc, bktName, objName2, uploadInfo.UploadID, 2, sourceCopy, 0, partSize)
|
||||
})
|
||||
|
||||
t.Run("check correct size when copy part from encrypted source", func(t *testing.T) {
|
||||
newBucket, newObjName := "new-bucket", "new-object-multipart"
|
||||
bktInfo := createTestBucket(hc, newBucket)
|
||||
|
||||
srcObjName := "source-object"
|
||||
key := []byte("firstencriptionkeyofsourceobject")
|
||||
keyMd5 := md5.Sum(key)
|
||||
srcEnc, err := encryption.NewParams(key)
|
||||
require.NoError(t, err)
|
||||
srcObjInfo := createTestObject(hc, bktInfo, srcObjName, *srcEnc)
|
||||
|
||||
multipartInfo := createMultipartUpload(hc, newBucket, newObjName, headers)
|
||||
|
||||
sourceCopy := newBucket + "/" + srcObjName
|
||||
|
||||
query := make(url.Values)
|
||||
query.Set(uploadIDQuery, multipartInfo.UploadID)
|
||||
query.Set(partNumberQuery, "1")
|
||||
|
||||
// empty copy-source-sse headers
|
||||
w, r := prepareTestRequestWithQuery(hc, newBucket, newObjName, query, nil)
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
r.Header.Set(api.AmzCopySource, sourceCopy)
|
||||
hc.Handler().UploadPartCopy(w, r)
|
||||
|
||||
assertStatus(t, w, http.StatusBadRequest)
|
||||
|
||||
// success copy
|
||||
w, r = prepareTestRequestWithQuery(hc, newBucket, newObjName, query, nil)
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
r.Header.Set(api.AmzCopySource, sourceCopy)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKey, base64.StdEncoding.EncodeToString(key))
|
||||
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMd5[:]))
|
||||
hc.Handler().UploadPartCopy(w, r)
|
||||
|
||||
uploadPartCopyResponse := &UploadPartCopyResponse{}
|
||||
readResponse(hc.t, w, http.StatusOK, uploadPartCopyResponse)
|
||||
|
||||
completeMultipartUpload(hc, newBucket, newObjName, multipartInfo.UploadID, []string{uploadPartCopyResponse.ETag})
|
||||
attr := getObjectAttributes(hc, newBucket, newObjName, objectParts)
|
||||
require.Equal(t, 1, attr.ObjectParts.PartsCount)
|
||||
require.Equal(t, srcObjInfo.Headers[layer.AttributeDecryptedSize], strconv.Itoa(attr.ObjectParts.Parts[0].Size))
|
||||
})
|
||||
}
|
||||
|
||||
func TestListParts(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-test-list-parts", "object-multipart"
|
||||
_ = createTestBucket(hc, bktName)
|
||||
partSize := 5 * 1024 * 1024
|
||||
|
||||
uploadInfo := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
||||
etag1, _ := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSize)
|
||||
etag2, _ := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 2, partSize)
|
||||
|
||||
list := listParts(hc, bktName, objName, uploadInfo.UploadID, "0", http.StatusOK)
|
||||
require.Len(t, list.Parts, 2)
|
||||
require.Equal(t, etag1, list.Parts[0].ETag)
|
||||
require.Equal(t, etag2, list.Parts[1].ETag)
|
||||
|
||||
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "1", http.StatusOK)
|
||||
require.Len(t, list.Parts, 1)
|
||||
require.Equal(t, etag2, list.Parts[0].ETag)
|
||||
|
||||
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "2", http.StatusOK)
|
||||
require.Len(t, list.Parts, 0)
|
||||
|
||||
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "7", http.StatusOK)
|
||||
require.Len(t, list.Parts, 0)
|
||||
|
||||
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "-1", http.StatusInternalServerError)
|
||||
require.Len(t, list.Parts, 0)
|
||||
}
|
||||
|
||||
func TestMultipartUploadWithContentLanguage(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-1", "object-1"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
partSize := 5 * 1024 * 1024
|
||||
exceptedContentLanguage := "en"
|
||||
headers := map[string]string{
|
||||
api.ContentLanguage: exceptedContentLanguage,
|
||||
}
|
||||
|
||||
multipartUpload := createMultipartUpload(hc, bktName, objName, headers)
|
||||
etag1, _ := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 1, partSize)
|
||||
etag2, _ := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 2, partSize)
|
||||
w := completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1, etag2})
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
||||
hc.Handler().HeadObjectHandler(w, r)
|
||||
require.Equal(t, exceptedContentLanguage, w.Header().Get(api.ContentLanguage))
|
||||
}
|
||||
|
||||
func TestMultipartUploadEnabledMD5(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
hc.config.md5Enabled = true
|
||||
hc.layerFeatures.SetMD5Enabled(true)
|
||||
|
||||
bktName, objName := "bucket-md5", "object-md5"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
partSize := 5 * 1024 * 1024
|
||||
multipartUpload := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
||||
etag1, partBody1 := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 1, partSize)
|
||||
md5Sum1 := md5.Sum(partBody1)
|
||||
require.Equal(t, data.Quote(hex.EncodeToString(md5Sum1[:])), etag1)
|
||||
|
||||
etag2, partBody2 := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 2, partSize)
|
||||
md5Sum2 := md5.Sum(partBody2)
|
||||
require.Equal(t, data.Quote(hex.EncodeToString(md5Sum2[:])), etag2)
|
||||
|
||||
w := completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1, etag2})
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
resp := &CompleteMultipartUploadResponse{}
|
||||
err := xml.NewDecoder(w.Result().Body).Decode(resp)
|
||||
require.NoError(t, err)
|
||||
completeMD5Sum := md5.Sum(append(md5Sum1[:], md5Sum2[:]...))
|
||||
require.Equal(t, data.Quote(hex.EncodeToString(completeMD5Sum[:])+"-2"), resp.ETag)
|
||||
}
|
||||
|
||||
func TestUploadPartCheckContentSHA256(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-1", "object-1"
|
||||
createTestBucket(hc, bktName)
|
||||
partSize := 5 * 1024 * 1024
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
hash string
|
||||
content []byte
|
||||
error bool
|
||||
}{
|
||||
{
|
||||
name: "invalid hash value",
|
||||
hash: "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
|
||||
content: []byte("content"),
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "correct hash for empty payload",
|
||||
hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
content: []byte(""),
|
||||
error: false,
|
||||
},
|
||||
{
|
||||
name: "unsigned payload",
|
||||
hash: "UNSIGNED-PAYLOAD",
|
||||
content: []byte("content"),
|
||||
error: false,
|
||||
},
|
||||
{
|
||||
name: "correct hash",
|
||||
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
|
||||
content: []byte("content"),
|
||||
error: false,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
multipartUpload := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
||||
|
||||
etag1, data1 := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 1, partSize)
|
||||
|
||||
query := make(url.Values)
|
||||
query.Set(uploadIDQuery, multipartUpload.UploadID)
|
||||
query.Set(partNumberQuery, strconv.Itoa(2))
|
||||
|
||||
w, r := prepareTestRequestWithQuery(hc, bktName, objName, query, tc.content)
|
||||
r.Header.Set(api.AmzContentSha256, tc.hash)
|
||||
hc.Handler().UploadPartHandler(w, r)
|
||||
if tc.error {
|
||||
assertS3Error(t, w, s3Errors.GetAPIError(s3Errors.ErrContentSHA256Mismatch))
|
||||
|
||||
list := listParts(hc, bktName, objName, multipartUpload.UploadID, "0", http.StatusOK)
|
||||
require.Len(t, list.Parts, 1)
|
||||
|
||||
w := completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1})
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
data, _ := getObject(hc, bktName, objName)
|
||||
equalDataSlices(t, data1, data)
|
||||
return
|
||||
}
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
list := listParts(hc, bktName, objName, multipartUpload.UploadID, "0", http.StatusOK)
|
||||
require.Len(t, list.Parts, 2)
|
||||
|
||||
etag2 := w.Header().Get(api.ETag)
|
||||
w = completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1, etag2})
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
data, _ := getObject(hc, bktName, objName)
|
||||
equalDataSlices(t, append(data1, tc.content...), data)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
|
||||
return uploadPartCopyBase(hc, bktName, objName, false, uploadID, num, srcObj, start, end)
|
||||
}
|
||||
|
||||
func uploadPartCopyBase(hc *handlerContext, bktName, objName string, encrypted bool, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
|
||||
query := make(url.Values)
|
||||
query.Set(uploadIDQuery, uploadID)
|
||||
query.Set(partNumberQuery, strconv.Itoa(num))
|
||||
|
||||
w, r := prepareTestRequestWithQuery(hc, bktName, objName, query, nil)
|
||||
if encrypted {
|
||||
setEncryptHeaders(r)
|
||||
}
|
||||
r.Header.Set(api.AmzCopySource, srcObj)
|
||||
if start+end > 0 {
|
||||
r.Header.Set(api.AmzCopySourceRange, fmt.Sprintf("bytes=%d-%d", start, end))
|
||||
}
|
||||
|
||||
hc.Handler().UploadPartCopy(w, r)
|
||||
uploadPartCopyResponse := &UploadPartCopyResponse{}
|
||||
readResponse(hc.t, w, http.StatusOK, uploadPartCopyResponse)
|
||||
|
||||
return uploadPartCopyResponse
|
||||
}
|
||||
|
||||
func listAllMultipartUploads(hc *handlerContext, bktName string) *ListMultipartUploadsResponse {
|
||||
return listMultipartUploadsBase(hc, bktName, "", "", "", "", -1)
|
||||
}
|
||||
|
||||
func listMultipartUploadsBase(hc *handlerContext, bktName, prefix, delimiter, uploadIDMarker, keyMarker string, maxUploads int) *ListMultipartUploadsResponse {
|
||||
query := make(url.Values)
|
||||
query.Set(prefixQueryName, prefix)
|
||||
query.Set(delimiterQueryName, delimiter)
|
||||
query.Set(uploadIDMarkerQueryName, uploadIDMarker)
|
||||
query.Set(keyMarkerQueryName, keyMarker)
|
||||
if maxUploads != -1 {
|
||||
query.Set(maxUploadsQueryName, strconv.Itoa(maxUploads))
|
||||
}
|
||||
|
||||
w, r := prepareTestRequestWithQuery(hc, bktName, "", query, nil)
|
||||
|
||||
hc.Handler().ListMultipartUploadsHandler(w, r)
|
||||
listPartsResponse := &ListMultipartUploadsResponse{}
|
||||
readResponse(hc.t, w, http.StatusOK, listPartsResponse)
|
||||
|
||||
return listPartsResponse
|
||||
}
|
||||
|
||||
func listParts(hc *handlerContext, bktName, objName string, uploadID, partNumberMarker string, status int) *ListPartsResponse {
|
||||
return listPartsBase(hc, bktName, objName, false, uploadID, partNumberMarker, status)
|
||||
}
|
||||
|
||||
func listPartsBase(hc *handlerContext, bktName, objName string, encrypted bool, uploadID, partNumberMarker string, status int) *ListPartsResponse {
|
||||
query := make(url.Values)
|
||||
query.Set(uploadIDQuery, uploadID)
|
||||
query.Set(partNumberMarkerQuery, partNumberMarker)
|
||||
|
||||
w, r := prepareTestRequestWithQuery(hc, bktName, objName, query, nil)
|
||||
if encrypted {
|
||||
setEncryptHeaders(r)
|
||||
}
|
||||
|
||||
hc.Handler().ListPartsHandler(w, r)
|
||||
listPartsResponse := &ListPartsResponse{}
|
||||
readResponse(hc.t, w, status, listPartsResponse)
|
||||
|
||||
return listPartsResponse
|
||||
}
|
|
@ -3,14 +3,18 @@ package handler
|
|||
import (
|
||||
"net/http"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
)
|
||||
|
||||
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||
}
|
||||
|
||||
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||
}
|
||||
|
||||
func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||
}
|
||||
|
|
|
@ -2,17 +2,17 @@ package handler
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
|
@ -21,10 +21,15 @@ type (
|
|||
Event string
|
||||
NotificationInfo *data.NotificationInfo
|
||||
BktInfo *data.BucketInfo
|
||||
ReqInfo *middleware.ReqInfo
|
||||
ReqInfo *api.ReqInfo
|
||||
User string
|
||||
Time time.Time
|
||||
}
|
||||
|
||||
NotificationConfiguration struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NotificationConfiguation"`
|
||||
NotificationConfiguration data.NotificationConfiguration
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -91,7 +96,7 @@ var validEvents = map[string]struct{}{
|
|||
}
|
||||
|
||||
func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
|
@ -99,7 +104,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
|
|||
}
|
||||
|
||||
conf := &data.NotificationConfiguration{}
|
||||
if err = h.cfg.NewXMLDecoder(r.Body).Decode(conf); err != nil {
|
||||
if err = xml.NewDecoder(r.Body).Decode(conf); err != nil {
|
||||
h.logAndSendError(w, "couldn't decode notification configuration", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
|
||||
return
|
||||
}
|
||||
|
@ -113,12 +118,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
|
|||
RequestInfo: reqInfo,
|
||||
BktInfo: bktInfo,
|
||||
Configuration: conf,
|
||||
}
|
||||
|
||||
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||
return
|
||||
CopiesNumber: h.cfg.CopiesNumber,
|
||||
}
|
||||
|
||||
if err = h.obj.PutBucketNotificationConfiguration(r.Context(), p); err != nil {
|
||||
|
@ -128,7 +128,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
|
|||
}
|
||||
|
||||
func (h *handler) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -142,14 +142,14 @@ func (h *handler) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Re
|
|||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, conf); err != nil {
|
||||
if err = api.EncodeToResponse(w, conf); err != nil {
|
||||
h.logAndSendError(w, "could not encode bucket notification configuration to response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) sendNotifications(ctx context.Context, p *SendNotificationParams) error {
|
||||
if !h.cfg.NotificatorEnabled() {
|
||||
if !h.cfg.NotificatorEnabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -161,7 +161,7 @@ func (h *handler) sendNotifications(ctx context.Context, p *SendNotificationPara
|
|||
return nil
|
||||
}
|
||||
|
||||
box, err := middleware.GetBoxData(ctx)
|
||||
box, err := layer.GetBoxData(ctx)
|
||||
if err == nil && box.Gate.BearerToken != nil {
|
||||
p.User = bearer.ResolveIssuer(*box.Gate.BearerToken).EncodeToString()
|
||||
}
|
||||
|
@ -174,7 +174,7 @@ func (h *handler) sendNotifications(ctx context.Context, p *SendNotificationPara
|
|||
}
|
||||
|
||||
// checkBucketConfiguration checks notification configuration and generates an ID for configurations with empty ids.
|
||||
func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.NotificationConfiguration, r *middleware.ReqInfo) (completed bool, err error) {
|
||||
func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.NotificationConfiguration, r *api.ReqInfo) (completed bool, err error) {
|
||||
if conf == nil {
|
||||
return
|
||||
}
|
||||
|
@ -192,12 +192,12 @@ func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.Notif
|
|||
return
|
||||
}
|
||||
|
||||
if h.cfg.NotificatorEnabled() {
|
||||
if h.cfg.NotificatorEnabled {
|
||||
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host, layer.TimeNow(ctx)); err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
h.reqLogger(ctx).Warn(logs.FailedToSendTestEventBecauseNotificationsIsDisabled)
|
||||
h.log.Warn("failed to send test event because notifications is disabled")
|
||||
}
|
||||
|
||||
if q.ID == "" {
|
||||
|
|
|
@ -3,8 +3,8 @@ package handler
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
|
@ -6,17 +6,16 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
// ListObjectsV1Handler handles objects listing requests for API version 1.
|
||||
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
params, err := parseListObjectsArgsV1(reqInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "failed to parse arguments", reqInfo, err)
|
||||
|
@ -34,12 +33,12 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, h.encodeV1(params, list)); err != nil {
|
||||
if err = api.EncodeToResponse(w, encodeV1(params, list)); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) *ListObjectsV1Response {
|
||||
func encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) *ListObjectsV1Response {
|
||||
res := &ListObjectsV1Response{
|
||||
Name: p.BktInfo.Name,
|
||||
EncodingType: p.Encode,
|
||||
|
@ -53,14 +52,14 @@ func (h *handler) encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjects
|
|||
|
||||
res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode)
|
||||
|
||||
res.Contents = fillContentsWithOwner(list.Objects, p.Encode, h.cfg.MD5Enabled())
|
||||
res.Contents = fillContentsWithOwner(list.Objects, p.Encode)
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// ListObjectsV2Handler handles objects listing requests for API version 2.
|
||||
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
params, err := parseListObjectsArgsV2(reqInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "failed to parse arguments", reqInfo, err)
|
||||
|
@ -78,12 +77,12 @@ func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, h.encodeV2(params, list)); err != nil {
|
||||
if err = api.EncodeToResponse(w, encodeV2(params, list)); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) *ListObjectsV2Response {
|
||||
func encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) *ListObjectsV2Response {
|
||||
res := &ListObjectsV2Response{
|
||||
Name: p.BktInfo.Name,
|
||||
EncodingType: p.Encode,
|
||||
|
@ -99,12 +98,12 @@ func (h *handler) encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjects
|
|||
|
||||
res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode)
|
||||
|
||||
res.Contents = fillContents(list.Objects, p.Encode, p.FetchOwner, h.cfg.MD5Enabled())
|
||||
res.Contents = fillContents(list.Objects, p.Encode, p.FetchOwner)
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func parseListObjectsArgsV1(reqInfo *middleware.ReqInfo) (*layer.ListObjectsParamsV1, error) {
|
||||
func parseListObjectsArgsV1(reqInfo *api.ReqInfo) (*layer.ListObjectsParamsV1, error) {
|
||||
var (
|
||||
res layer.ListObjectsParamsV1
|
||||
queryValues = reqInfo.URL.Query()
|
||||
|
@ -121,7 +120,7 @@ func parseListObjectsArgsV1(reqInfo *middleware.ReqInfo) (*layer.ListObjectsPara
|
|||
return &res, nil
|
||||
}
|
||||
|
||||
func parseListObjectsArgsV2(reqInfo *middleware.ReqInfo) (*layer.ListObjectsParamsV2, error) {
|
||||
func parseListObjectsArgsV2(reqInfo *api.ReqInfo) (*layer.ListObjectsParamsV2, error) {
|
||||
var (
|
||||
res layer.ListObjectsParamsV2
|
||||
queryValues = reqInfo.URL.Query()
|
||||
|
@ -143,7 +142,7 @@ func parseListObjectsArgsV2(reqInfo *middleware.ReqInfo) (*layer.ListObjectsPara
|
|||
return &res, nil
|
||||
}
|
||||
|
||||
func parseListObjectArgs(reqInfo *middleware.ReqInfo) (*layer.ListObjectsParamsCommon, error) {
|
||||
func parseListObjectArgs(reqInfo *api.ReqInfo) (*layer.ListObjectsParamsCommon, error) {
|
||||
var (
|
||||
err error
|
||||
res layer.ListObjectsParamsCommon
|
||||
|
@ -185,26 +184,24 @@ func fillPrefixes(src []string, encode string) []CommonPrefix {
|
|||
return dst
|
||||
}
|
||||
|
||||
func fillContentsWithOwner(src []*data.ExtendedNodeVersion, encode string, md5Enabled bool) []Object {
|
||||
return fillContents(src, encode, true, md5Enabled)
|
||||
func fillContentsWithOwner(src []*data.ObjectInfo, encode string) []Object {
|
||||
return fillContents(src, encode, true)
|
||||
}
|
||||
|
||||
func fillContents(src []*data.ExtendedNodeVersion, encode string, fetchOwner, md5Enabled bool) []Object {
|
||||
func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) []Object {
|
||||
var dst []Object
|
||||
for _, obj := range src {
|
||||
res := Object{
|
||||
Key: s3PathEncode(obj.NodeVersion.FilePath, encode),
|
||||
Size: obj.NodeVersion.Size,
|
||||
LastModified: obj.NodeVersion.Created.UTC().Format(time.RFC3339),
|
||||
ETag: data.Quote(obj.NodeVersion.GetETag(md5Enabled)),
|
||||
StorageClass: api.DefaultStorageClass,
|
||||
Key: s3PathEncode(obj.Name, encode),
|
||||
Size: obj.Size,
|
||||
LastModified: obj.Created.UTC().Format(time.RFC3339),
|
||||
ETag: obj.HashSum,
|
||||
}
|
||||
|
||||
if fetchOwner {
|
||||
owner := obj.NodeVersion.Owner.String()
|
||||
res.Owner = &Owner{
|
||||
ID: owner,
|
||||
DisplayName: owner,
|
||||
ID: obj.Owner.String(),
|
||||
DisplayName: obj.Owner.String(),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -214,7 +211,7 @@ func fillContents(src []*data.ExtendedNodeVersion, encode string, fetchOwner, md
|
|||
}
|
||||
|
||||
func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
p, err := parseListObjectVersionsRequest(reqInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "failed to parse request", reqInfo, err)
|
||||
|
@ -232,13 +229,13 @@ func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http
|
|||
return
|
||||
}
|
||||
|
||||
response := encodeListObjectVersionsToResponse(info, p.BktInfo.Name, h.cfg.MD5Enabled())
|
||||
if err = middleware.EncodeToResponse(w, response); err != nil {
|
||||
response := encodeListObjectVersionsToResponse(info, p.BktInfo.Name)
|
||||
if err = api.EncodeToResponse(w, response); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func parseListObjectVersionsRequest(reqInfo *middleware.ReqInfo) (*layer.ListObjectVersionsParams, error) {
|
||||
func parseListObjectVersionsRequest(reqInfo *api.ReqInfo) (*layer.ListObjectVersionsParams, error) {
|
||||
var (
|
||||
err error
|
||||
res layer.ListObjectVersionsParams
|
||||
|
@ -252,19 +249,15 @@ func parseListObjectVersionsRequest(reqInfo *middleware.ReqInfo) (*layer.ListObj
|
|||
}
|
||||
|
||||
res.Prefix = queryValues.Get("prefix")
|
||||
res.KeyMarker = queryValues.Get("key-marker")
|
||||
res.KeyMarker = queryValues.Get("marker")
|
||||
res.Delimiter = queryValues.Get("delimiter")
|
||||
res.Encode = queryValues.Get("encoding-type")
|
||||
res.VersionIDMarker = queryValues.Get("version-id-marker")
|
||||
|
||||
if res.VersionIDMarker != "" && res.KeyMarker == "" {
|
||||
return nil, errors.GetAPIError(errors.VersionIDMarkerWithoutKeyMarker)
|
||||
}
|
||||
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, bucketName string, md5Enabled bool) *ListObjectsVersionsResponse {
|
||||
func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, bucketName string) *ListObjectsVersionsResponse {
|
||||
res := ListObjectsVersionsResponse{
|
||||
Name: bucketName,
|
||||
IsTruncated: info.IsTruncated,
|
||||
|
@ -281,27 +274,26 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck
|
|||
for _, ver := range info.Version {
|
||||
res.Version = append(res.Version, ObjectVersionResponse{
|
||||
IsLatest: ver.IsLatest,
|
||||
Key: ver.NodeVersion.FilePath,
|
||||
LastModified: ver.NodeVersion.Created.UTC().Format(time.RFC3339),
|
||||
Key: ver.ObjectInfo.Name,
|
||||
LastModified: ver.ObjectInfo.Created.UTC().Format(time.RFC3339),
|
||||
Owner: Owner{
|
||||
ID: ver.NodeVersion.Owner.String(),
|
||||
DisplayName: ver.NodeVersion.Owner.String(),
|
||||
ID: ver.ObjectInfo.Owner.String(),
|
||||
DisplayName: ver.ObjectInfo.Owner.String(),
|
||||
},
|
||||
Size: ver.NodeVersion.Size,
|
||||
VersionID: ver.Version(),
|
||||
ETag: data.Quote(ver.NodeVersion.GetETag(md5Enabled)),
|
||||
StorageClass: api.DefaultStorageClass,
|
||||
Size: ver.ObjectInfo.Size,
|
||||
VersionID: ver.Version(),
|
||||
ETag: ver.ObjectInfo.HashSum,
|
||||
})
|
||||
}
|
||||
// this loop is not starting till versioning is not implemented
|
||||
for _, del := range info.DeleteMarker {
|
||||
res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{
|
||||
IsLatest: del.IsLatest,
|
||||
Key: del.NodeVersion.FilePath,
|
||||
LastModified: del.NodeVersion.Created.UTC().Format(time.RFC3339),
|
||||
Key: del.ObjectInfo.Name,
|
||||
LastModified: del.ObjectInfo.Created.UTC().Format(time.RFC3339),
|
||||
Owner: Owner{
|
||||
ID: del.NodeVersion.Owner.String(),
|
||||
DisplayName: del.NodeVersion.Owner.String(),
|
||||
ID: del.ObjectInfo.Owner.String(),
|
||||
DisplayName: del.ObjectInfo.Owner.String(),
|
||||
},
|
||||
VersionID: del.Version(),
|
||||
})
|
||||
|
|
|
@ -1,22 +1,13 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap/zaptest"
|
||||
)
|
||||
|
||||
func TestParseContinuationToken(t *testing.T) {
|
||||
|
@ -65,246 +56,6 @@ func TestListObjectNullVersions(t *testing.T) {
|
|||
require.Equal(t, data.UnversionedObjectVersionID, result.Version[1].VersionID)
|
||||
}
|
||||
|
||||
func TestListObjectsWithOldTreeNodes(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-versioning-enabled", "object"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
srcEnc, err := encryption.NewParams([]byte("1234567890qwertyuiopasdfghjklzxc"))
|
||||
require.NoError(t, err)
|
||||
|
||||
n := 10
|
||||
objInfos := make([]*data.ObjectInfo, n)
|
||||
for i := 0; i < n; i++ {
|
||||
objInfos[i] = createTestObject(hc, bktInfo, objName+strconv.Itoa(i), *srcEnc)
|
||||
}
|
||||
sort.Slice(objInfos, func(i, j int) bool { return objInfos[i].Name < objInfos[j].Name })
|
||||
|
||||
makeAllTreeObjectsOld(hc, bktInfo)
|
||||
|
||||
listV1 := listObjectsV1(hc, bktName, "", "", "", -1)
|
||||
checkListOldNodes(hc, listV1.Contents, objInfos)
|
||||
|
||||
listV2 := listObjectsV2(hc, bktName, "", "", "", "", -1)
|
||||
checkListOldNodes(hc, listV2.Contents, objInfos)
|
||||
|
||||
listVers := listObjectsVersions(hc, bktName, "", "", "", "", -1)
|
||||
checkListVersionsOldNodes(hc, listVers.Version, objInfos)
|
||||
}
|
||||
|
||||
func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
|
||||
nodes, err := hc.treeMock.GetSubTree(hc.Context(), bktInfo, "version", 0, 0)
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
for _, node := range nodes {
|
||||
if node.GetNodeID() == 0 {
|
||||
continue
|
||||
}
|
||||
meta := make(map[string]string, len(node.GetMeta()))
|
||||
for _, m := range node.GetMeta() {
|
||||
if m.GetKey() != "Created" && m.GetKey() != "Owner" {
|
||||
meta[m.GetKey()] = string(m.GetValue())
|
||||
}
|
||||
}
|
||||
|
||||
err = hc.treeMock.MoveNode(hc.Context(), bktInfo, "version", node.GetNodeID(), node.GetParentID(), meta)
|
||||
require.NoError(hc.t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkListOldNodes(hc *handlerContext, list []Object, objInfos []*data.ObjectInfo) {
|
||||
require.Len(hc.t, list, len(objInfos))
|
||||
for i := range list {
|
||||
require.Equal(hc.t, objInfos[i].Name, list[i].Key)
|
||||
realSize, err := layer.GetObjectSize(objInfos[i])
|
||||
require.NoError(hc.t, err)
|
||||
require.Equal(hc.t, objInfos[i].Owner.EncodeToString(), list[i].Owner.ID)
|
||||
require.Equal(hc.t, objInfos[i].Created.UTC().Format(time.RFC3339), list[i].LastModified)
|
||||
require.Equal(hc.t, realSize, list[i].Size)
|
||||
}
|
||||
}
|
||||
|
||||
func checkListVersionsOldNodes(hc *handlerContext, list []ObjectVersionResponse, objInfos []*data.ObjectInfo) {
|
||||
require.Len(hc.t, list, len(objInfos))
|
||||
for i := range list {
|
||||
require.Equal(hc.t, objInfos[i].Name, list[i].Key)
|
||||
realSize, err := layer.GetObjectSize(objInfos[i])
|
||||
require.NoError(hc.t, err)
|
||||
require.Equal(hc.t, objInfos[i].Owner.EncodeToString(), list[i].Owner.ID)
|
||||
require.Equal(hc.t, objInfos[i].Created.UTC().Format(time.RFC3339), list[i].LastModified)
|
||||
require.Equal(hc.t, realSize, list[i].Size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListObjectsContextCanceled(t *testing.T) {
|
||||
layerCfg := layer.DefaultCachesConfigs(zaptest.NewLogger(t))
|
||||
layerCfg.SessionList.Lifetime = time.Hour
|
||||
layerCfg.SessionList.Size = 1
|
||||
|
||||
hc := prepareHandlerContextBase(t, layerCfg)
|
||||
|
||||
bktName := "bucket-versioning-enabled"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
for i := 0; i < 4; i++ {
|
||||
putObject(hc, bktName, "object"+strconv.Itoa(i))
|
||||
}
|
||||
|
||||
result := listObjectsV1(hc, bktName, "", "", "", 2)
|
||||
session := hc.cache.GetListSession(hc.owner, cache.CreateListSessionCacheKey(bktInfo.CID, "", result.NextMarker))
|
||||
// invoke list again to trigger cache eviction
|
||||
// (use empty prefix to check that context canceled on replace)
|
||||
listObjectsV1(hc, bktName, "", "", "", 2)
|
||||
checkContextCanceled(session.Context, t)
|
||||
|
||||
result2 := listObjectsV2(hc, bktName, "", "", "", "", 2)
|
||||
session2 := hc.cache.GetListSession(hc.owner, cache.CreateListSessionCacheKey(bktInfo.CID, "", result2.NextContinuationToken))
|
||||
// invoke list again to trigger cache eviction
|
||||
// (use non-empty prefix to check that context canceled on cache eviction)
|
||||
listObjectsV2(hc, bktName, "o", "", "", "", 2)
|
||||
checkContextCanceled(session2.Context, t)
|
||||
|
||||
result3 := listObjectsVersions(hc, bktName, "", "", "", "", 2)
|
||||
session3 := hc.cache.GetListSession(hc.owner, cache.CreateListSessionCacheKey(bktInfo.CID, "", result3.NextVersionIDMarker))
|
||||
// invoke list again to trigger cache eviction
|
||||
listObjectsVersions(hc, bktName, "o", "", "", "", 2)
|
||||
checkContextCanceled(session3.Context, t)
|
||||
}
|
||||
|
||||
func checkContextCanceled(ctx context.Context, t *testing.T) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
require.ErrorIs(t, ctx.Err(), context.Canceled)
|
||||
}
|
||||
|
||||
func TestListObjectsLatestVersions(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-versioning-enabled"
|
||||
createTestBucket(hc, bktName)
|
||||
putBucketVersioning(t, hc, bktName, true)
|
||||
|
||||
objName1, objName2 := "object1", "object2"
|
||||
objContent1, objContent2 := "content1", "content2"
|
||||
|
||||
putObjectContent(hc, bktName, objName1, objContent1)
|
||||
hdr1 := putObjectContent(hc, bktName, objName1, objContent2)
|
||||
putObjectContent(hc, bktName, objName2, objContent1)
|
||||
hdr2 := putObjectContent(hc, bktName, objName2, objContent2)
|
||||
|
||||
t.Run("listv1", func(t *testing.T) {
|
||||
result := listObjectsV1(hc, bktName, "", "", "", -1)
|
||||
|
||||
require.Len(t, result.Contents, 2)
|
||||
require.Equal(t, objName1, result.Contents[0].Key)
|
||||
require.Equal(t, hdr1.Get(api.ETag), result.Contents[0].ETag)
|
||||
require.Equal(t, objName2, result.Contents[1].Key)
|
||||
require.Equal(t, hdr2.Get(api.ETag), result.Contents[1].ETag)
|
||||
})
|
||||
|
||||
t.Run("listv2", func(t *testing.T) {
|
||||
result := listObjectsV2(hc, bktName, "", "", "", "", -1)
|
||||
|
||||
require.Len(t, result.Contents, 2)
|
||||
require.Equal(t, objName1, result.Contents[0].Key)
|
||||
require.Equal(t, hdr1.Get(api.ETag), result.Contents[0].ETag)
|
||||
require.Equal(t, objName2, result.Contents[1].Key)
|
||||
require.Equal(t, hdr2.Get(api.ETag), result.Contents[1].ETag)
|
||||
})
|
||||
}
|
||||
|
||||
func TestListObjectsVersionsPaging(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-versioning-enabled"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
n := 12
|
||||
|
||||
var objects []string
|
||||
for i := 0; i < n; i++ {
|
||||
objects = append(objects, "objects"+strconv.Itoa(i))
|
||||
putObjectContent(hc, bktName, objects[i], "content")
|
||||
}
|
||||
sort.Strings(objects)
|
||||
|
||||
result := &ListObjectsVersionsResponse{IsTruncated: true}
|
||||
for result.IsTruncated {
|
||||
result = listObjectsVersions(hc, bktName, "", "", result.NextKeyMarker, result.NextVersionIDMarker, n/3)
|
||||
|
||||
for i, version := range result.Version {
|
||||
if objects[i] != version.Key {
|
||||
t.Errorf("expected: '%s', got: '%s'", objects[i], version.Key)
|
||||
}
|
||||
}
|
||||
objects = objects[len(result.Version):]
|
||||
}
|
||||
|
||||
require.Empty(t, objects)
|
||||
}
|
||||
|
||||
func TestListObjectsVersionsCorrectIsLatestFlag(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-versioning-enabled"
|
||||
createVersionedBucket(hc, bktName)
|
||||
|
||||
objName1, objName2 := "obj1", "obj2"
|
||||
|
||||
n := 9
|
||||
listSize := 3
|
||||
headers := make([]http.Header, n)
|
||||
// objects uploaded: ["obj1"-v1, "obj1"-v2, "obj1"-v3, "obj2"-v1, "obj2"-v2, "obj2"-v3, "obj2"-v4, "obj2"-v5, "obj2"-v6]
|
||||
for i := 0; i < n; i++ {
|
||||
objName := objName1
|
||||
if i >= listSize {
|
||||
objName = objName2
|
||||
}
|
||||
headers[i] = putObjectContent(hc, bktName, objName, fmt.Sprintf("content/%d", i))
|
||||
}
|
||||
|
||||
versions := listObjectsVersions(hc, bktName, "", "", "", "", listSize)
|
||||
// expected objects: ["obj1"-v3, "obj1"-v2, "obj1"-v1]
|
||||
checkListVersionsParts(t, versions, formReverseVersionResponse(objName1, headers[:listSize], true))
|
||||
|
||||
versions = listObjectsVersions(hc, bktName, "", "", versions.NextKeyMarker, versions.NextVersionIDMarker, listSize)
|
||||
// expected objects: ["obj2"-v6, "obj2"-v5, "obj2"-v4]
|
||||
checkListVersionsParts(t, versions, formReverseVersionResponse(objName2, headers[2*listSize:], true))
|
||||
|
||||
versions = listObjectsVersions(hc, bktName, "", "", versions.NextKeyMarker, versions.NextVersionIDMarker, listSize)
|
||||
// expected objects: ["obj2"-v3, "obj2"-v2, "obj2"-v1]
|
||||
checkListVersionsParts(t, versions, formReverseVersionResponse(objName2, headers[listSize:2*listSize], false))
|
||||
}
|
||||
|
||||
func formReverseVersionResponse(objName string, headers []http.Header, isLatest bool) []ObjectVersionResponse {
|
||||
res := make([]ObjectVersionResponse, len(headers))
|
||||
|
||||
for i, h := range headers {
|
||||
ind := len(headers) - 1 - i
|
||||
res[ind] = ObjectVersionResponse{
|
||||
ETag: h.Get(api.ETag),
|
||||
IsLatest: isLatest && ind == 0,
|
||||
Key: objName,
|
||||
VersionID: h.Get(api.AmzVersionID),
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func checkListVersionsParts(t *testing.T, versions *ListObjectsVersionsResponse, expected []ObjectVersionResponse) {
|
||||
require.Len(t, versions.Version, len(expected))
|
||||
for i, res := range versions.Version {
|
||||
require.Equal(t, expected[i].Key, res.Key)
|
||||
require.Equal(t, expected[i].ETag, res.ETag)
|
||||
require.Equal(t, expected[i].VersionID, res.VersionID)
|
||||
require.Equal(t, expected[i].IsLatest, res.IsLatest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3CompatibilityBucketListV2BothContinuationTokenStartAfter(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
|
@ -313,14 +64,14 @@ func TestS3CompatibilityBucketListV2BothContinuationTokenStartAfter(t *testing.T
|
|||
bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])
|
||||
|
||||
for _, objName := range objects[1:] {
|
||||
createTestObject(tc, bktInfo, objName, encryption.Params{})
|
||||
createTestObject(tc, bktInfo, objName)
|
||||
}
|
||||
|
||||
listV2Response1 := listObjectsV2(tc, bktName, "", "", "bar", "", 1)
|
||||
listV2Response1 := listObjectsV2(t, tc, bktName, "", "", "bar", "", 1)
|
||||
nextContinuationToken := listV2Response1.NextContinuationToken
|
||||
require.Equal(t, "baz", listV2Response1.Contents[0].Key)
|
||||
|
||||
listV2Response2 := listObjectsV2(tc, bktName, "", "", "bar", nextContinuationToken, -1)
|
||||
listV2Response2 := listObjectsV2(t, tc, bktName, "", "", "bar", nextContinuationToken, -1)
|
||||
|
||||
require.Equal(t, nextContinuationToken, listV2Response2.ContinuationToken)
|
||||
require.Equal(t, "bar", listV2Response2.StartAfter)
|
||||
|
@ -330,36 +81,6 @@ func TestS3CompatibilityBucketListV2BothContinuationTokenStartAfter(t *testing.T
|
|||
require.Equal(t, "quxx", listV2Response2.Contents[1].Key)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2EncodingBasic(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing-v1-encoding"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"foo+1/bar", "foo/bar/xyzzy", "quux ab/thud", "asdf+b"}
|
||||
for _, objName := range objects {
|
||||
createTestObject(hc, bktInfo, objName, encryption.Params{})
|
||||
}
|
||||
|
||||
query := make(url.Values)
|
||||
query.Add("delimiter", "/")
|
||||
query.Add("encoding-type", "url")
|
||||
|
||||
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
||||
hc.Handler().ListObjectsV2Handler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
listV2Response := &ListObjectsV2Response{}
|
||||
parseTestResponse(hc.t, w, listV2Response)
|
||||
|
||||
require.Equal(t, "/", listV2Response.Delimiter)
|
||||
require.Len(t, listV2Response.Contents, 1)
|
||||
require.Equal(t, "asdf%2Bb", listV2Response.Contents[0].Key)
|
||||
require.Len(t, listV2Response.CommonPrefixes, 3)
|
||||
require.Equal(t, "foo%2B1/", listV2Response.CommonPrefixes[0].Prefix)
|
||||
require.Equal(t, "foo/", listV2Response.CommonPrefixes[1].Prefix)
|
||||
require.Equal(t, "quux%20ab/", listV2Response.CommonPrefixes[2].Prefix)
|
||||
}
|
||||
|
||||
func TestS3BucketListDelimiterBasic(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
|
@ -368,10 +89,10 @@ func TestS3BucketListDelimiterBasic(t *testing.T) {
|
|||
bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])
|
||||
|
||||
for _, objName := range objects[1:] {
|
||||
createTestObject(tc, bktInfo, objName, encryption.Params{})
|
||||
createTestObject(tc, bktInfo, objName)
|
||||
}
|
||||
|
||||
listV1Response := listObjectsV1(tc, bktName, "", "/", "", -1)
|
||||
listV1Response := listObjectsV1(t, tc, bktName, "", "/", "", -1)
|
||||
require.Equal(t, "/", listV1Response.Delimiter)
|
||||
require.Equal(t, "asdf", listV1Response.Contents[0].Key)
|
||||
require.Len(t, listV1Response.CommonPrefixes, 2)
|
||||
|
@ -379,181 +100,6 @@ func TestS3BucketListDelimiterBasic(t *testing.T) {
|
|||
require.Equal(t, "quux/", listV1Response.CommonPrefixes[1].Prefix)
|
||||
}
|
||||
|
||||
func TestS3BucketListEmpty(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
versions := listObjectsVersions(hc, bktName, "", "", "", "", -1)
|
||||
require.Empty(t, versions.Version)
|
||||
require.Empty(t, versions.DeleteMarker)
|
||||
require.Empty(t, versions.CommonPrefixes)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixAlt(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"bar", "baz", "foo"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "ba", "", "", "", -1)
|
||||
|
||||
require.Equal(t, "ba", response.Prefix)
|
||||
require.Len(t, response.Contents, 2)
|
||||
require.Equal(t, "bar", response.Contents[0].Key)
|
||||
require.Equal(t, "baz", response.Contents[1].Key)
|
||||
require.Empty(t, response.CommonPrefixes)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixNotExist(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"foo/bar", "foo/baz", "quux"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "d", "", "", "", -1)
|
||||
|
||||
require.Equal(t, "d", response.Prefix)
|
||||
require.Empty(t, response.Contents)
|
||||
require.Empty(t, response.CommonPrefixes)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixUnreadable(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"foo/bar", "foo/baz", "quux"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "\x0a", "", "", "", -1)
|
||||
|
||||
require.Equal(t, "\x0a", response.Prefix)
|
||||
require.Empty(t, response.Contents)
|
||||
require.Empty(t, response.CommonPrefixes)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixDelimiterAlt(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"bar", "bazar", "cab", "foo"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "ba", "a", "", "", -1)
|
||||
|
||||
require.Equal(t, "ba", response.Prefix)
|
||||
require.Equal(t, "a", response.Delimiter)
|
||||
require.Len(t, response.Contents, 1)
|
||||
require.Equal(t, "bar", response.Contents[0].Key)
|
||||
require.Len(t, response.CommonPrefixes, 1)
|
||||
require.Equal(t, "baza", response.CommonPrefixes[0].Prefix)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixDelimiterDelimiterNotExist(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"b/a/c", "b/a/g", "b/a/r", "g"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "b", "z", "", "", -1)
|
||||
|
||||
require.Len(t, response.Contents, 3)
|
||||
require.Equal(t, "b/a/c", response.Contents[0].Key)
|
||||
require.Equal(t, "b/a/g", response.Contents[1].Key)
|
||||
require.Equal(t, "b/a/r", response.Contents[2].Key)
|
||||
require.Empty(t, response.CommonPrefixes)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixDelimiterPrefixDelimiterNotExist(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"b/a/c", "b/a/g", "b/a/r", "g"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "y", "z", "", "", -1)
|
||||
|
||||
require.Empty(t, response.Contents)
|
||||
require.Empty(t, response.CommonPrefixes)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2DelimiterPercentage(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
objects := []string{"b%ar", "b%az", "c%ab", "foo"}
|
||||
bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])
|
||||
|
||||
for _, objName := range objects[1:] {
|
||||
createTestObject(tc, bktInfo, objName, encryption.Params{})
|
||||
}
|
||||
|
||||
listV2Response := listObjectsV2(tc, bktName, "", "%", "", "", -1)
|
||||
require.Equal(t, "%", listV2Response.Delimiter)
|
||||
require.Len(t, listV2Response.Contents, 1)
|
||||
require.Equal(t, "foo", listV2Response.Contents[0].Key)
|
||||
require.Len(t, listV2Response.CommonPrefixes, 2)
|
||||
require.Equal(t, "b%", listV2Response.CommonPrefixes[0].Prefix)
|
||||
require.Equal(t, "c%", listV2Response.CommonPrefixes[1].Prefix)
|
||||
}
|
||||
|
||||
func TestS3BucketListDelimiterPrefix(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"asdf", "boo/bar", "boo/baz/xyzzy", "cquux/thud", "cquux/bla"}
|
||||
for _, objName := range objects {
|
||||
createTestObject(hc, bktInfo, objName, encryption.Params{})
|
||||
}
|
||||
|
||||
var empty []string
|
||||
delim := "/"
|
||||
prefix := ""
|
||||
|
||||
marker := validateListV1(t, hc, bktName, prefix, delim, "", 1, true, []string{"asdf"}, empty, "asdf")
|
||||
marker = validateListV1(t, hc, bktName, prefix, delim, marker, 1, true, empty, []string{"boo/"}, "boo/")
|
||||
validateListV1(t, hc, bktName, prefix, delim, marker, 1, false, empty, []string{"cquux/"}, "")
|
||||
|
||||
marker = validateListV1(t, hc, bktName, prefix, delim, "", 2, true, []string{"asdf"}, []string{"boo/"}, "boo/")
|
||||
validateListV1(t, hc, bktName, prefix, delim, marker, 2, false, empty, []string{"cquux/"}, "")
|
||||
|
||||
prefix = "boo/"
|
||||
marker = validateListV1(t, hc, bktName, prefix, delim, "", 1, true, []string{"boo/bar"}, empty, "boo/bar")
|
||||
validateListV1(t, hc, bktName, prefix, delim, marker, 1, false, empty, []string{"boo/baz/"}, "")
|
||||
|
||||
validateListV1(t, hc, bktName, prefix, delim, "", 2, false, []string{"boo/bar"}, []string{"boo/baz/"}, "")
|
||||
}
|
||||
|
||||
func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
|
@ -562,7 +108,7 @@ func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
|
|||
bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])
|
||||
|
||||
for _, objName := range objects[1:] {
|
||||
createTestObject(tc, bktInfo, objName, encryption.Params{})
|
||||
createTestObject(tc, bktInfo, objName)
|
||||
}
|
||||
|
||||
var empty []string
|
||||
|
@ -583,152 +129,26 @@ func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
|
|||
validateListV2(t, tc, bktName, prefix, delim, "", 2, false, true, []string{"boo/bar"}, []string{"boo/baz/"})
|
||||
}
|
||||
|
||||
func TestS3BucketListDelimiterPrefixUnderscore(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"_obj1_", "_under1/bar", "_under1/baz/xyzzy", "_under2/thud", "_under2/bla"}
|
||||
for _, objName := range objects {
|
||||
createTestObject(hc, bktInfo, objName, encryption.Params{})
|
||||
}
|
||||
|
||||
var empty []string
|
||||
delim := "/"
|
||||
prefix := ""
|
||||
|
||||
marker := validateListV1(t, hc, bktName, prefix, delim, "", 1, true, []string{"_obj1_"}, empty, "_obj1_")
|
||||
marker = validateListV1(t, hc, bktName, prefix, delim, marker, 1, true, empty, []string{"_under1/"}, "_under1/")
|
||||
validateListV1(t, hc, bktName, prefix, delim, marker, 1, false, empty, []string{"_under2/"}, "")
|
||||
|
||||
marker = validateListV1(t, hc, bktName, prefix, delim, "", 2, true, []string{"_obj1_"}, []string{"_under1/"}, "_under1/")
|
||||
validateListV1(t, hc, bktName, prefix, delim, marker, 2, false, empty, []string{"_under2/"}, "")
|
||||
|
||||
prefix = "_under1/"
|
||||
marker = validateListV1(t, hc, bktName, prefix, delim, "", 1, true, []string{"_under1/bar"}, empty, "_under1/bar")
|
||||
validateListV1(t, hc, bktName, prefix, delim, marker, 1, false, empty, []string{"_under1/baz/"}, "")
|
||||
|
||||
validateListV1(t, hc, bktName, prefix, delim, "", 2, false, []string{"_under1/bar"}, []string{"_under1/baz/"}, "")
|
||||
}
|
||||
|
||||
func TestS3BucketListDelimiterNotSkipSpecial(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"0/"}
|
||||
for i := 1000; i < 1999; i++ {
|
||||
objects = append(objects, fmt.Sprintf("0/%d", i))
|
||||
}
|
||||
|
||||
objects2 := []string{"1999", "1999#", "1999+", "2000"}
|
||||
objects = append(objects, objects2...)
|
||||
|
||||
for _, objName := range objects {
|
||||
createTestObject(hc, bktInfo, objName, encryption.Params{})
|
||||
}
|
||||
|
||||
delimiter := "/"
|
||||
list := listObjectsV1(hc, bktName, "", delimiter, "", -1)
|
||||
|
||||
require.Equal(t, delimiter, list.Delimiter)
|
||||
require.Equal(t, []CommonPrefix{{Prefix: "0/"}}, list.CommonPrefixes)
|
||||
|
||||
require.Len(t, list.Contents, len(objects2))
|
||||
for i := 0; i < len(list.Contents); i++ {
|
||||
require.Equal(t, objects2[i], list.Contents[i].Key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMintVersioningListObjectVersionsVersionIDContinuation(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "mint-bucket-for-listing-versions", "objName"
|
||||
createTestBucket(hc, bktName)
|
||||
putBucketVersioning(t, hc, bktName, true)
|
||||
|
||||
length := 10
|
||||
objects := make([]string, length)
|
||||
for i := 0; i < length; i++ {
|
||||
objects[i] = objName
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
maxKeys := 5
|
||||
|
||||
page1 := listObjectsVersions(hc, bktName, "", "", "", "", maxKeys)
|
||||
require.Len(t, page1.Version, maxKeys)
|
||||
checkVersionsNames(t, page1, objects)
|
||||
require.Equal(t, page1.Version[maxKeys-1].VersionID, page1.NextVersionIDMarker)
|
||||
require.True(t, page1.IsTruncated)
|
||||
require.Empty(t, page1.KeyMarker)
|
||||
require.Empty(t, page1.VersionIDMarker)
|
||||
|
||||
page2 := listObjectsVersions(hc, bktName, "", "", page1.NextKeyMarker, page1.NextVersionIDMarker, maxKeys)
|
||||
require.Len(t, page2.Version, maxKeys)
|
||||
checkVersionsNames(t, page1, objects)
|
||||
require.Empty(t, page2.NextVersionIDMarker)
|
||||
require.False(t, page2.IsTruncated)
|
||||
require.Equal(t, page1.NextKeyMarker, page2.KeyMarker)
|
||||
require.Equal(t, page1.NextVersionIDMarker, page2.VersionIDMarker)
|
||||
}
|
||||
|
||||
func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, names []string) {
|
||||
for i, v := range versions.Version {
|
||||
require.Equal(t, names[i], v.Key)
|
||||
}
|
||||
}
|
||||
|
||||
func listObjectsV2(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
|
||||
return listObjectsV2Ext(hc, bktName, prefix, delimiter, startAfter, continuationToken, "", maxKeys)
|
||||
}
|
||||
|
||||
func listObjectsV2Ext(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken, encodingType string, maxKeys int) *ListObjectsV2Response {
|
||||
func listObjectsV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
|
||||
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
||||
query.Add("fetch-owner", "true")
|
||||
if len(startAfter) != 0 {
|
||||
query.Add("start-after", startAfter)
|
||||
}
|
||||
if len(continuationToken) != 0 {
|
||||
query.Add("continuation-token", continuationToken)
|
||||
}
|
||||
if len(encodingType) != 0 {
|
||||
query.Add("encoding-type", encodingType)
|
||||
}
|
||||
|
||||
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
||||
hc.Handler().ListObjectsV2Handler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
w, r := prepareTestFullRequest(tc, bktName, "", query, nil)
|
||||
tc.Handler().ListObjectsV2Handler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
res := &ListObjectsV2Response{}
|
||||
parseTestResponse(hc.t, w, res)
|
||||
parseTestResponse(t, w, res)
|
||||
return res
|
||||
}
|
||||
|
||||
func validateListV1(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int,
|
||||
isTruncated bool, checkObjects, checkPrefixes []string, nextMarker string) string {
|
||||
response := listObjectsV1(tc, bktName, prefix, delimiter, marker, maxKeys)
|
||||
|
||||
require.Equal(t, isTruncated, response.IsTruncated)
|
||||
require.Equal(t, nextMarker, response.NextMarker)
|
||||
|
||||
require.Len(t, response.Contents, len(checkObjects))
|
||||
for i := 0; i < len(checkObjects); i++ {
|
||||
require.Equal(t, checkObjects[i], response.Contents[i].Key)
|
||||
}
|
||||
|
||||
require.Len(t, response.CommonPrefixes, len(checkPrefixes))
|
||||
for i := 0; i < len(checkPrefixes); i++ {
|
||||
require.Equal(t, checkPrefixes[i], response.CommonPrefixes[i].Prefix)
|
||||
}
|
||||
|
||||
return response.NextMarker
|
||||
}
|
||||
|
||||
func validateListV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, continuationToken string, maxKeys int,
|
||||
isTruncated, last bool, checkObjects, checkPrefixes []string) string {
|
||||
response := listObjectsV2(tc, bktName, prefix, delimiter, "", continuationToken, maxKeys)
|
||||
response := listObjectsV2(t, tc, bktName, prefix, delimiter, "", continuationToken, maxKeys)
|
||||
|
||||
require.Equal(t, isTruncated, response.IsTruncated)
|
||||
require.Equal(t, last, len(response.NextContinuationToken) == 0)
|
||||
|
@ -762,33 +182,16 @@ func prepareCommonListObjectsQuery(prefix, delimiter string, maxKeys int) url.Va
|
|||
return query
|
||||
}
|
||||
|
||||
func listObjectsV1(hc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response {
|
||||
func listObjectsV1(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response {
|
||||
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
||||
if len(marker) != 0 {
|
||||
query.Add("marker", marker)
|
||||
}
|
||||
|
||||
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
||||
hc.Handler().ListObjectsV1Handler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
w, r := prepareTestFullRequest(tc, bktName, "", query, nil)
|
||||
tc.Handler().ListObjectsV1Handler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
res := &ListObjectsV1Response{}
|
||||
parseTestResponse(hc.t, w, res)
|
||||
return res
|
||||
}
|
||||
|
||||
func listObjectsVersions(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker string, maxKeys int) *ListObjectsVersionsResponse {
|
||||
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
||||
if len(keyMarker) != 0 {
|
||||
query.Add("key-marker", keyMarker)
|
||||
}
|
||||
if len(versionIDMarker) != 0 {
|
||||
query.Add("version-id-marker", versionIDMarker)
|
||||
}
|
||||
|
||||
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
||||
hc.Handler().ListBucketObjectVersionsHandler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
res := &ListObjectsVersionsResponse{}
|
||||
parseTestResponse(hc.t, w, res)
|
||||
parseTestResponse(t, w, res)
|
||||
return res
|
||||
}
|
||||
|
|
|
@ -4,10 +4,9 @@ import (
|
|||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
stderrors "errors"
|
||||
errorsStd "errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
|
@ -17,22 +16,15 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/schema/native"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/schema/s3"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -51,13 +43,13 @@ func (p *postPolicy) condition(key string) *policyCondition {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (p *postPolicy) CheckContentLength(size uint64) bool {
|
||||
func (p *postPolicy) CheckContentLength(size int64) bool {
|
||||
if p.empty {
|
||||
return true
|
||||
}
|
||||
for _, condition := range p.Conditions {
|
||||
if condition.Matching == "content-length-range" {
|
||||
length := strconv.FormatUint(size, 10)
|
||||
length := strconv.FormatInt(size, 10)
|
||||
return condition.Key <= length && length <= condition.Value
|
||||
}
|
||||
}
|
||||
|
@ -186,31 +178,11 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
err error
|
||||
newEaclTable *eacl.Table
|
||||
sessionTokenEACL *session.Container
|
||||
cannedACLStatus = aclHeadersStatus(r)
|
||||
ctx = r.Context()
|
||||
reqInfo = middleware.GetReqInfo(ctx)
|
||||
containsACL = containsACLHeaders(r)
|
||||
reqInfo = api.GetReqInfo(r.Context())
|
||||
)
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket objInfo", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
apeEnabled := bktInfo.APEEnabled || settings.CannedACL != ""
|
||||
if apeEnabled && cannedACLStatus == aclStatusYes {
|
||||
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
||||
return
|
||||
}
|
||||
|
||||
needUpdateEACLTable := !(apeEnabled || cannedACLStatus == aclStatusNo)
|
||||
if needUpdateEACLTable {
|
||||
if containsACL {
|
||||
if sessionTokenEACL, err = getSessionTokenSetEACL(r.Context()); err != nil {
|
||||
h.logAndSendError(w, "could not get eacl session token from a box", reqInfo, err)
|
||||
return
|
||||
|
@ -223,6 +195,12 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket objInfo", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
metadata := parseMetadata(r)
|
||||
if contentType := r.Header.Get(api.ContentType); len(contentType) > 0 {
|
||||
metadata[api.ContentType] = contentType
|
||||
|
@ -233,8 +211,11 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
if expires := r.Header.Get(api.Expires); len(expires) > 0 {
|
||||
metadata[api.Expires] = expires
|
||||
}
|
||||
if contentLanguage := r.Header.Get(api.ContentLanguage); len(contentLanguage) > 0 {
|
||||
metadata[api.ContentLanguage] = contentLanguage
|
||||
|
||||
copiesNumber, err := getCopiesNumberOrDefault(metadata, h.cfg.CopiesNumber)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
encryptionParams, err := formEncryptionParams(r)
|
||||
|
@ -243,47 +224,32 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
body, err := h.getBodyReader(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "failed to get body reader", reqInfo, err)
|
||||
return
|
||||
}
|
||||
if encodings := r.Header.Get(api.ContentEncoding); len(encodings) > 0 {
|
||||
metadata[api.ContentEncoding] = encodings
|
||||
}
|
||||
|
||||
var size uint64
|
||||
if r.ContentLength > 0 {
|
||||
size = uint64(r.ContentLength)
|
||||
}
|
||||
|
||||
params := &layer.PutObjectParams{
|
||||
BktInfo: bktInfo,
|
||||
Object: reqInfo.ObjectName,
|
||||
Reader: body,
|
||||
Size: size,
|
||||
Header: metadata,
|
||||
Encryption: encryptionParams,
|
||||
ContentMD5: r.Header.Get(api.ContentMD5),
|
||||
ContentSHA256Hash: r.Header.Get(api.AmzContentSha256),
|
||||
BktInfo: bktInfo,
|
||||
Object: reqInfo.ObjectName,
|
||||
Reader: r.Body,
|
||||
Size: r.ContentLength,
|
||||
Header: metadata,
|
||||
Encryption: encryptionParams,
|
||||
CopiesNumber: copiesNumber,
|
||||
}
|
||||
|
||||
params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
params.Lock, err = formObjectLock(ctx, bktInfo, settings.LockConfiguration, r.Header)
|
||||
params.Lock, err = formObjectLock(r.Context(), bktInfo, settings.LockConfiguration, r.Header)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not form object lock", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
extendedObjInfo, err := h.obj.PutObject(ctx, params)
|
||||
extendedObjInfo, err := h.obj.PutObject(r.Context(), params)
|
||||
if err != nil {
|
||||
_, err2 := io.Copy(io.Discard, body)
|
||||
err3 := body.Close()
|
||||
_, err2 := io.Copy(io.Discard, r.Body)
|
||||
err3 := r.Body.Close()
|
||||
h.logAndSendError(w, "could not upload object", reqInfo, err, zap.Errors("body close errors", []error{err2, err3}))
|
||||
return
|
||||
}
|
||||
|
@ -291,15 +257,15 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
s := &SendNotificationParams{
|
||||
Event: EventObjectCreatedPut,
|
||||
NotificationInfo: data.NotificationInfoFromObject(objInfo, h.cfg.MD5Enabled()),
|
||||
NotificationInfo: data.NotificationInfoFromObject(objInfo),
|
||||
BktInfo: bktInfo,
|
||||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
if err = h.sendNotifications(r.Context(), s); err != nil {
|
||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||
}
|
||||
|
||||
if needUpdateEACLTable {
|
||||
if containsACL {
|
||||
if newEaclTable, err = h.getNewEAclTable(r, bktInfo, objInfo); err != nil {
|
||||
h.logAndSendError(w, "could not get new eacl table", reqInfo, err)
|
||||
return
|
||||
|
@ -307,8 +273,8 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
if tagSet != nil {
|
||||
tagPrm := &data.PutObjectTaggingParams{
|
||||
ObjectVersion: &data.ObjectVersion{
|
||||
tagPrm := &layer.PutObjectTaggingParams{
|
||||
ObjectVersion: &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: objInfo.Name,
|
||||
VersionID: objInfo.VersionID(),
|
||||
|
@ -342,89 +308,35 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
addSSECHeaders(w.Header(), r.Header)
|
||||
}
|
||||
|
||||
w.Header().Set(api.ETag, data.Quote(objInfo.ETag(h.cfg.MD5Enabled())))
|
||||
|
||||
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
|
||||
h.logAndSendError(w, "write response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set(api.ETag, objInfo.HashSum)
|
||||
api.WriteSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
func (h *handler) getBodyReader(r *http.Request) (io.ReadCloser, error) {
|
||||
if !api.IsSignedStreamingV4(r) {
|
||||
return r.Body, nil
|
||||
func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber uint32) (uint32, error) {
|
||||
copiesNumberStr, ok := metadata[layer.AttributeFrostfsCopiesNumber]
|
||||
if !ok {
|
||||
return defaultCopiesNumber, nil
|
||||
}
|
||||
|
||||
encodings := r.Header.Values(api.ContentEncoding)
|
||||
var chunkedEncoding bool
|
||||
resultContentEncoding := make([]string, 0, len(encodings))
|
||||
for _, enc := range encodings {
|
||||
for _, e := range strings.Split(enc, ",") {
|
||||
e = strings.TrimSpace(e)
|
||||
if e == api.AwsChunked { // probably we should also check position of this header value
|
||||
chunkedEncoding = true
|
||||
} else {
|
||||
resultContentEncoding = append(resultContentEncoding, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
r.Header.Set(api.ContentEncoding, strings.Join(resultContentEncoding, ","))
|
||||
|
||||
if !chunkedEncoding && !h.cfg.BypassContentEncodingInChunks() {
|
||||
return nil, fmt.Errorf("%w: request is not chunk encoded, encodings '%s'",
|
||||
errors.GetAPIError(errors.ErrInvalidEncodingMethod), strings.Join(encodings, ","))
|
||||
}
|
||||
|
||||
decodeContentSize := r.Header.Get(api.AmzDecodedContentLength)
|
||||
if len(decodeContentSize) == 0 {
|
||||
return nil, errors.GetAPIError(errors.ErrMissingContentLength)
|
||||
}
|
||||
|
||||
if _, err := strconv.Atoi(decodeContentSize); err != nil {
|
||||
return nil, fmt.Errorf("%w: parse decoded content length: %s", errors.GetAPIError(errors.ErrMissingContentLength), err.Error())
|
||||
}
|
||||
|
||||
chunkReader, err := newSignV4ChunkedReader(r)
|
||||
copiesNumber, err := strconv.ParseUint(copiesNumberStr, 10, 32)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("initialize chunk reader: %w", err)
|
||||
return 0, fmt.Errorf("pasrse copies number: %w", err)
|
||||
}
|
||||
|
||||
return chunkReader, nil
|
||||
return uint32(copiesNumber), nil
|
||||
}
|
||||
|
||||
func formEncryptionParams(r *http.Request) (enc encryption.Params, err error) {
|
||||
return formEncryptionParamsBase(r, false)
|
||||
}
|
||||
|
||||
func formCopySourceEncryptionParams(r *http.Request) (enc encryption.Params, err error) {
|
||||
return formEncryptionParamsBase(r, true)
|
||||
}
|
||||
|
||||
func formEncryptionParamsBase(r *http.Request, isCopySource bool) (enc encryption.Params, err error) {
|
||||
var sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMD5 string
|
||||
if isCopySource {
|
||||
sseCustomerAlgorithm = r.Header.Get(api.AmzCopySourceServerSideEncryptionCustomerAlgorithm)
|
||||
sseCustomerKey = r.Header.Get(api.AmzCopySourceServerSideEncryptionCustomerKey)
|
||||
sseCustomerKeyMD5 = r.Header.Get(api.AmzCopySourceServerSideEncryptionCustomerKeyMD5)
|
||||
} else {
|
||||
sseCustomerAlgorithm = r.Header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
|
||||
sseCustomerKey = r.Header.Get(api.AmzServerSideEncryptionCustomerKey)
|
||||
sseCustomerKeyMD5 = r.Header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
|
||||
}
|
||||
sseCustomerAlgorithm := r.Header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
|
||||
sseCustomerKey := r.Header.Get(api.AmzServerSideEncryptionCustomerKey)
|
||||
sseCustomerKeyMD5 := r.Header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
|
||||
|
||||
if len(sseCustomerAlgorithm) == 0 && len(sseCustomerKey) == 0 && len(sseCustomerKeyMD5) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if r.TLS == nil {
|
||||
return enc, errors.GetAPIError(errors.ErrInsecureSSECustomerRequest)
|
||||
}
|
||||
|
||||
if len(sseCustomerKey) > 0 && len(sseCustomerAlgorithm) == 0 {
|
||||
return enc, errors.GetAPIError(errors.ErrMissingSSECustomerAlgorithm)
|
||||
}
|
||||
if len(sseCustomerAlgorithm) > 0 && len(sseCustomerKey) == 0 {
|
||||
return enc, errors.GetAPIError(errors.ErrMissingSSECustomerKey)
|
||||
return enc, errorsStd.New("encryption available only when TLS is enabled")
|
||||
}
|
||||
|
||||
if sseCustomerAlgorithm != layer.AESEncryptionAlgorithm {
|
||||
|
@ -433,16 +345,10 @@ func formEncryptionParamsBase(r *http.Request, isCopySource bool) (enc encryptio
|
|||
|
||||
key, err := base64.StdEncoding.DecodeString(sseCustomerKey)
|
||||
if err != nil {
|
||||
if isCopySource {
|
||||
return enc, errors.GetAPIError(errors.ErrInvalidSSECustomerParameters)
|
||||
}
|
||||
return enc, errors.GetAPIError(errors.ErrInvalidSSECustomerKey)
|
||||
}
|
||||
|
||||
if len(key) != layer.AESKeySize {
|
||||
if isCopySource {
|
||||
return enc, errors.GetAPIError(errors.ErrInvalidSSECustomerParameters)
|
||||
}
|
||||
return enc, errors.GetAPIError(errors.ErrInvalidSSECustomerKey)
|
||||
}
|
||||
|
||||
|
@ -469,10 +375,9 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
newEaclTable *eacl.Table
|
||||
tagSet map[string]string
|
||||
sessionTokenEACL *session.Container
|
||||
ctx = r.Context()
|
||||
reqInfo = middleware.GetReqInfo(ctx)
|
||||
reqInfo = api.GetReqInfo(r.Context())
|
||||
metadata = make(map[string]string)
|
||||
cannedACLStatus = aclHeadersStatus(r)
|
||||
containsACL = containsACLHeaders(r)
|
||||
)
|
||||
|
||||
policy, err := checkPostPolicy(r, reqInfo, metadata)
|
||||
|
@ -483,50 +388,25 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
if tagging := auth.MultipartFormValue(r, "tagging"); tagging != "" {
|
||||
buffer := bytes.NewBufferString(tagging)
|
||||
tags := new(data.Tagging)
|
||||
if err = h.cfg.NewXMLDecoder(buffer).Decode(tags); err != nil {
|
||||
h.logAndSendError(w, "could not decode tag set", reqInfo,
|
||||
fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
|
||||
return
|
||||
}
|
||||
tagSet, err = h.readTagSet(tags)
|
||||
tagSet, err = readTagSet(buffer)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not read tag set", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket objInfo", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
apeEnabled := bktInfo.APEEnabled || settings.CannedACL != ""
|
||||
if apeEnabled && cannedACLStatus == aclStatusYes {
|
||||
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
||||
return
|
||||
}
|
||||
|
||||
needUpdateEACLTable := !(apeEnabled || cannedACLStatus == aclStatusNo)
|
||||
if needUpdateEACLTable {
|
||||
if sessionTokenEACL, err = getSessionTokenSetEACL(ctx); err != nil {
|
||||
if containsACL {
|
||||
if sessionTokenEACL, err = getSessionTokenSetEACL(r.Context()); err != nil {
|
||||
h.logAndSendError(w, "could not get eacl session token from a box", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var contentReader io.Reader
|
||||
var size uint64
|
||||
var size int64
|
||||
if content, ok := r.MultipartForm.Value["file"]; ok {
|
||||
contentReader = bytes.NewBufferString(content[0])
|
||||
size = uint64(len(content[0]))
|
||||
size = int64(len(content[0]))
|
||||
} else {
|
||||
file, head, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
|
@ -534,7 +414,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
contentReader = file
|
||||
size = uint64(head.Size)
|
||||
size = head.Size
|
||||
reqInfo.ObjectName = strings.ReplaceAll(reqInfo.ObjectName, "${filename}", head.Filename)
|
||||
}
|
||||
if !policy.CheckContentLength(size) {
|
||||
|
@ -542,6 +422,12 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
params := &layer.PutObjectParams{
|
||||
BktInfo: bktInfo,
|
||||
Object: reqInfo.ObjectName,
|
||||
|
@ -550,7 +436,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
Header: metadata,
|
||||
}
|
||||
|
||||
extendedObjInfo, err := h.obj.PutObject(ctx, params)
|
||||
extendedObjInfo, err := h.obj.PutObject(r.Context(), params)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not upload object", reqInfo, err)
|
||||
return
|
||||
|
@ -559,12 +445,12 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
s := &SendNotificationParams{
|
||||
Event: EventObjectCreatedPost,
|
||||
NotificationInfo: data.NotificationInfoFromObject(objInfo, h.cfg.MD5Enabled()),
|
||||
NotificationInfo: data.NotificationInfoFromObject(objInfo),
|
||||
BktInfo: bktInfo,
|
||||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
if err = h.sendNotifications(r.Context(), s); err != nil {
|
||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||
}
|
||||
|
||||
if acl := auth.MultipartFormValue(r, "acl"); acl != "" {
|
||||
|
@ -580,8 +466,8 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
if tagSet != nil {
|
||||
tagPrm := &data.PutObjectTaggingParams{
|
||||
ObjectVersion: &data.ObjectVersion{
|
||||
tagPrm := &layer.PutObjectTaggingParams{
|
||||
ObjectVersion: &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: objInfo.Name,
|
||||
VersionID: objInfo.VersionID(),
|
||||
|
@ -589,7 +475,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
NodeVersion: extendedObjInfo.NodeVersion,
|
||||
}
|
||||
|
||||
if _, err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
|
||||
if _, err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
|
||||
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
@ -602,13 +488,15 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
SessionToken: sessionTokenEACL,
|
||||
}
|
||||
|
||||
if err = h.obj.PutBucketACL(ctx, p); err != nil {
|
||||
if err = h.obj.PutBucketACL(r.Context(), p); err != nil {
|
||||
h.logAndSendError(w, "could not put bucket acl", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if settings.VersioningEnabled() {
|
||||
if settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo); err != nil {
|
||||
h.log.Warn("couldn't get bucket versioning", zap.String("bucket name", reqInfo.BucketName), zap.Error(err))
|
||||
} else if settings.VersioningEnabled() {
|
||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
||||
}
|
||||
|
||||
|
@ -626,25 +514,21 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
resp := &PostResponse{
|
||||
Bucket: objInfo.Bucket,
|
||||
Key: objInfo.Name,
|
||||
ETag: data.Quote(objInfo.ETag(h.cfg.MD5Enabled())),
|
||||
ETag: objInfo.HashSum,
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
respData, err := middleware.EncodeResponse(resp)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "encode response", reqInfo, err)
|
||||
}
|
||||
if _, err = w.Write(respData); err != nil {
|
||||
if _, err = w.Write(api.EncodeResponse(resp)); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set(api.ETag, data.Quote(objInfo.ETag(h.cfg.MD5Enabled())))
|
||||
w.Header().Set(api.ETag, objInfo.HashSum)
|
||||
w.WriteHeader(status)
|
||||
}
|
||||
|
||||
func checkPostPolicy(r *http.Request, reqInfo *middleware.ReqInfo, metadata map[string]string) (*postPolicy, error) {
|
||||
func checkPostPolicy(r *http.Request, reqInfo *api.ReqInfo, metadata map[string]string) (*postPolicy, error) {
|
||||
policy := &postPolicy{empty: true}
|
||||
if policyStr := auth.MultipartFormValue(r, "policy"); policyStr != "" {
|
||||
policyData, err := base64.StdEncoding.DecodeString(policyStr)
|
||||
|
@ -660,21 +544,11 @@ func checkPostPolicy(r *http.Request, reqInfo *middleware.ReqInfo, metadata map[
|
|||
policy.empty = false
|
||||
}
|
||||
|
||||
if r.MultipartForm == nil {
|
||||
return nil, stderrors.New("empty multipart form")
|
||||
}
|
||||
|
||||
for key, v := range r.MultipartForm.Value {
|
||||
value := v[0]
|
||||
if key == "file" || key == "policy" || key == "x-amz-signature" || strings.HasPrefix(key, "x-ignore-") {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(v) != 1 {
|
||||
return nil, fmt.Errorf("empty multipart value for key '%s'", key)
|
||||
}
|
||||
|
||||
value := v[0]
|
||||
|
||||
if err := policy.CheckField(key, value); err != nil {
|
||||
return nil, fmt.Errorf("'%s' form field doesn't match the policy: %w", key, err)
|
||||
}
|
||||
|
@ -704,33 +578,9 @@ func checkPostPolicy(r *http.Request, reqInfo *middleware.ReqInfo, metadata map[
|
|||
return policy, nil
|
||||
}
|
||||
|
||||
type aclStatus int
|
||||
|
||||
const (
|
||||
// aclStatusNo means no acl headers at all.
|
||||
aclStatusNo aclStatus = iota
|
||||
// aclStatusYesAPECompatible means that only X-Amz-Acl present and equals to private.
|
||||
aclStatusYesAPECompatible
|
||||
// aclStatusYes means any other acl headers configuration.
|
||||
aclStatusYes
|
||||
)
|
||||
|
||||
func aclHeadersStatus(r *http.Request) aclStatus {
|
||||
if r.Header.Get(api.AmzGrantRead) != "" ||
|
||||
r.Header.Get(api.AmzGrantFullControl) != "" ||
|
||||
r.Header.Get(api.AmzGrantWrite) != "" {
|
||||
return aclStatusYes
|
||||
}
|
||||
|
||||
cannedACL := r.Header.Get(api.AmzACL)
|
||||
if cannedACL != "" {
|
||||
if cannedACL == basicACLPrivate {
|
||||
return aclStatusYesAPECompatible
|
||||
}
|
||||
return aclStatusYes
|
||||
}
|
||||
|
||||
return aclStatusNo
|
||||
func containsACLHeaders(r *http.Request) bool {
|
||||
return r.Header.Get(api.AmzACL) != "" || r.Header.Get(api.AmzGrantRead) != "" ||
|
||||
r.Header.Get(api.AmzGrantFullControl) != "" || r.Header.Get(api.AmzGrantWrite) != ""
|
||||
}
|
||||
|
||||
func (h *handler) getNewEAclTable(r *http.Request, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) (*eacl.Table, error) {
|
||||
|
@ -795,7 +645,7 @@ func parseTaggingHeader(header http.Header) (map[string]string, error) {
|
|||
}
|
||||
tagSet = make(map[string]string, len(queries))
|
||||
for k, v := range queries {
|
||||
tag := data.Tag{Key: k, Value: v[0]}
|
||||
tag := Tag{Key: k, Value: v[0]}
|
||||
if err = checkTag(tag); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -816,143 +666,20 @@ func parseMetadata(r *http.Request) map[string]string {
|
|||
return res
|
||||
}
|
||||
|
||||
func parseCannedACL(header http.Header) (string, error) {
|
||||
acl := header.Get(api.AmzACL)
|
||||
if len(acl) == 0 {
|
||||
return basicACLPrivate, nil
|
||||
}
|
||||
|
||||
if acl == basicACLPrivate || acl == basicACLPublic ||
|
||||
acl == cannedACLAuthRead || acl == basicACLReadOnly {
|
||||
return acl, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("unknown acl: %s", acl)
|
||||
}
|
||||
|
||||
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if h.cfg.ACLEnabled() {
|
||||
h.createBucketHandlerACL(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
h.createBucketHandlerPolicy(w, r)
|
||||
}
|
||||
|
||||
func (h *handler) parseCommonCreateBucketParams(reqInfo *middleware.ReqInfo, boxData *accessbox.Box, r *http.Request) (*keys.PublicKey, *layer.CreateBucketParams, error) {
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
p := &layer.CreateBucketParams{
|
||||
Name: reqInfo.BucketName,
|
||||
Namespace: reqInfo.Namespace,
|
||||
SessionContainerCreation: boxData.Gate.SessionTokenForPut(),
|
||||
}
|
||||
|
||||
if p.SessionContainerCreation == nil {
|
||||
return nil, nil, fmt.Errorf("%w: couldn't find session token for put", errors.GetAPIError(errors.ErrAccessDenied))
|
||||
Name: reqInfo.BucketName,
|
||||
}
|
||||
|
||||
if err := checkBucketName(reqInfo.BucketName); err != nil {
|
||||
return nil, nil, fmt.Errorf("invalid bucket name: %w", err)
|
||||
h.logAndSendError(w, "invalid bucket name", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
key, err := getTokenIssuerKey(boxData)
|
||||
key, err := h.bearerTokenIssuerKey(r.Context())
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("couldn't get bearer token signature key: %w", err)
|
||||
}
|
||||
|
||||
createParams, err := h.parseLocationConstraint(r)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("could not parse location contraint: %w", err)
|
||||
}
|
||||
|
||||
if err = h.setPlacementPolicy(p, reqInfo.Namespace, createParams.LocationConstraint, boxData.Policies); err != nil {
|
||||
return nil, nil, fmt.Errorf("couldn't set placement policy: %w", err)
|
||||
}
|
||||
|
||||
p.ObjectLockEnabled = isLockEnabled(h.reqLogger(r.Context()), r.Header)
|
||||
|
||||
return key, p, nil
|
||||
}
|
||||
|
||||
func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
|
||||
boxData, err := middleware.GetBoxData(ctx)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "get access box from request", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
key, p, err := h.parseCommonCreateBucketParams(reqInfo, boxData, r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "parse create bucket params", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
cannedACL, err := parseCannedACL(r.Header)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not parse canned ACL", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
p.APEEnabled = true
|
||||
bktInfo, err := h.obj.CreateBucket(ctx, p)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not create bucket", reqInfo, err)
|
||||
return
|
||||
}
|
||||
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
|
||||
|
||||
chains := bucketCannedACLToAPERules(cannedACL, reqInfo, key, bktInfo.CID)
|
||||
if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chains); err != nil {
|
||||
h.logAndSendError(w, "failed to add morph rule chain", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
sp := &layer.PutSettingsParams{
|
||||
BktInfo: bktInfo,
|
||||
Settings: &data.BucketSettings{
|
||||
CannedACL: cannedACL,
|
||||
OwnerKey: key,
|
||||
Versioning: data.VersioningUnversioned,
|
||||
},
|
||||
}
|
||||
|
||||
if p.ObjectLockEnabled {
|
||||
sp.Settings.Versioning = data.VersioningEnabled
|
||||
}
|
||||
|
||||
if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
|
||||
h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
|
||||
zap.String("container_id", bktInfo.CID.EncodeToString()))
|
||||
return
|
||||
}
|
||||
|
||||
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
|
||||
h.logAndSendError(w, "write response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) createBucketHandlerACL(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
|
||||
boxData, err := middleware.GetBoxData(ctx)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "get access box from request", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
key, p, err := h.parseCommonCreateBucketParams(reqInfo, boxData, r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "parse create bucket params", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
aclPrm := &layer.PutBucketACLParams{SessionToken: boxData.Gate.SessionTokenForSetEACL()}
|
||||
if aclPrm.SessionToken == nil {
|
||||
h.logAndSendError(w, "couldn't find session token for setEACL", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
|
||||
h.logAndSendError(w, "couldn't get bearer token signature key", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -963,179 +690,69 @@ func (h *handler) createBucketHandlerACL(w http.ResponseWriter, r *http.Request)
|
|||
}
|
||||
resInfo := &resourceInfo{Bucket: reqInfo.BucketName}
|
||||
|
||||
aclPrm.EACL, err = bucketACLToTable(bktACL, resInfo)
|
||||
p.EACL, err = bucketACLToTable(bktACL, resInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could translate bucket acl to eacl", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
bktInfo, err := h.obj.CreateBucket(ctx, p)
|
||||
createParams, err := parseLocationConstraint(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not parse body", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
var policies []*accessbox.ContainerPolicy
|
||||
boxData, err := layer.GetBoxData(r.Context())
|
||||
if err == nil {
|
||||
policies = boxData.Policies
|
||||
p.SessionContainerCreation = boxData.Gate.SessionTokenForPut()
|
||||
p.SessionEACL = boxData.Gate.SessionTokenForSetEACL()
|
||||
}
|
||||
|
||||
if p.SessionContainerCreation == nil {
|
||||
h.logAndSendError(w, "couldn't find session token for put", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
|
||||
return
|
||||
}
|
||||
|
||||
if p.SessionEACL == nil {
|
||||
h.logAndSendError(w, "couldn't find session token for setEACL", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
|
||||
return
|
||||
}
|
||||
|
||||
if err = h.setPolicy(p, createParams.LocationConstraint, policies); err != nil {
|
||||
h.logAndSendError(w, "couldn't set placement policy", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
p.ObjectLockEnabled = isLockEnabled(r.Header)
|
||||
|
||||
bktInfo, err := h.obj.CreateBucket(r.Context(), p)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not create bucket", reqInfo, err)
|
||||
return
|
||||
}
|
||||
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
|
||||
|
||||
aclPrm.BktInfo = bktInfo
|
||||
if err = h.obj.PutBucketACL(r.Context(), aclPrm); err != nil {
|
||||
h.logAndSendError(w, "could not put bucket e/ACL", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
sp := &layer.PutSettingsParams{
|
||||
BktInfo: bktInfo,
|
||||
Settings: &data.BucketSettings{
|
||||
OwnerKey: key,
|
||||
Versioning: data.VersioningUnversioned,
|
||||
},
|
||||
}
|
||||
h.log.Info("bucket is created", zap.String("reqId", reqInfo.RequestID),
|
||||
zap.String("bucket", reqInfo.BucketName), zap.Stringer("container_id", bktInfo.CID))
|
||||
|
||||
if p.ObjectLockEnabled {
|
||||
sp.Settings.Versioning = data.VersioningEnabled
|
||||
sp := &layer.PutSettingsParams{
|
||||
BktInfo: bktInfo,
|
||||
Settings: &data.BucketSettings{Versioning: data.VersioningEnabled},
|
||||
}
|
||||
if err = h.obj.PutBucketSettings(r.Context(), sp); err != nil {
|
||||
h.logAndSendError(w, "couldn't enable bucket versioning", reqInfo, err,
|
||||
zap.String("container_id", bktInfo.CID.EncodeToString()))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
|
||||
h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
|
||||
zap.String("container_id", bktInfo.CID.EncodeToString()))
|
||||
return
|
||||
}
|
||||
|
||||
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
|
||||
h.logAndSendError(w, "write response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
api.WriteSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
const s3ActionPrefix = "s3:"
|
||||
|
||||
var (
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html
|
||||
|
||||
writeACLBucketS3Actions = []string{
|
||||
s3ActionPrefix + middleware.PutObjectOperation,
|
||||
s3ActionPrefix + middleware.PostObjectOperation,
|
||||
s3ActionPrefix + middleware.CopyObjectOperation,
|
||||
s3ActionPrefix + middleware.UploadPartOperation,
|
||||
s3ActionPrefix + middleware.UploadPartCopyOperation,
|
||||
s3ActionPrefix + middleware.CreateMultipartUploadOperation,
|
||||
s3ActionPrefix + middleware.CompleteMultipartUploadOperation,
|
||||
}
|
||||
|
||||
readACLBucketS3Actions = []string{
|
||||
s3ActionPrefix + middleware.HeadBucketOperation,
|
||||
s3ActionPrefix + middleware.GetBucketLocationOperation,
|
||||
s3ActionPrefix + middleware.ListObjectsV1Operation,
|
||||
s3ActionPrefix + middleware.ListObjectsV2Operation,
|
||||
s3ActionPrefix + middleware.ListBucketObjectVersionsOperation,
|
||||
s3ActionPrefix + middleware.ListMultipartUploadsOperation,
|
||||
}
|
||||
|
||||
writeACLBucketNativeActions = []string{
|
||||
native.MethodPutObject,
|
||||
}
|
||||
|
||||
readACLBucketNativeActions = []string{
|
||||
native.MethodGetContainer,
|
||||
native.MethodGetObject,
|
||||
native.MethodHeadObject,
|
||||
native.MethodSearchObject,
|
||||
native.MethodRangeObject,
|
||||
native.MethodHashObject,
|
||||
}
|
||||
)
|
||||
|
||||
func bucketCannedACLToAPERules(cannedACL string, reqInfo *middleware.ReqInfo, key *keys.PublicKey, cnrID cid.ID) []*chain.Chain {
|
||||
cnrIDStr := cnrID.EncodeToString()
|
||||
|
||||
chains := []*chain.Chain{
|
||||
{
|
||||
ID: getBucketCannedChainID(chain.S3, cnrID),
|
||||
Rules: []chain.Rule{{
|
||||
Status: chain.Allow,
|
||||
Actions: chain.Actions{Names: []string{"s3:*"}},
|
||||
Resources: chain.Resources{Names: []string{
|
||||
fmt.Sprintf(s3.ResourceFormatS3Bucket, reqInfo.BucketName),
|
||||
fmt.Sprintf(s3.ResourceFormatS3BucketObjects, reqInfo.BucketName),
|
||||
}},
|
||||
Condition: []chain.Condition{{
|
||||
Op: chain.CondStringEquals,
|
||||
Object: chain.ObjectRequest,
|
||||
Key: s3.PropertyKeyOwner,
|
||||
Value: key.Address(),
|
||||
}},
|
||||
}}},
|
||||
{
|
||||
ID: getBucketCannedChainID(chain.Ingress, cnrID),
|
||||
Rules: []chain.Rule{{
|
||||
Status: chain.Allow,
|
||||
Actions: chain.Actions{Names: []string{"*"}},
|
||||
Resources: chain.Resources{Names: []string{
|
||||
fmt.Sprintf(native.ResourceFormatNamespaceContainer, reqInfo.Namespace, cnrIDStr),
|
||||
fmt.Sprintf(native.ResourceFormatNamespaceContainerObjects, reqInfo.Namespace, cnrIDStr),
|
||||
}},
|
||||
Condition: []chain.Condition{{
|
||||
Op: chain.CondStringEquals,
|
||||
Object: chain.ObjectRequest,
|
||||
Key: native.PropertyKeyActorPublicKey,
|
||||
Value: hex.EncodeToString(key.Bytes()),
|
||||
}},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
switch cannedACL {
|
||||
case basicACLPrivate:
|
||||
case cannedACLAuthRead:
|
||||
fallthrough
|
||||
case basicACLReadOnly:
|
||||
chains[0].Rules = append(chains[0].Rules, chain.Rule{
|
||||
Status: chain.Allow,
|
||||
Actions: chain.Actions{Names: readACLBucketS3Actions},
|
||||
Resources: chain.Resources{Names: []string{
|
||||
fmt.Sprintf(s3.ResourceFormatS3Bucket, reqInfo.BucketName),
|
||||
fmt.Sprintf(s3.ResourceFormatS3BucketObjects, reqInfo.BucketName),
|
||||
}},
|
||||
})
|
||||
|
||||
chains[1].Rules = append(chains[1].Rules, chain.Rule{
|
||||
Status: chain.Allow,
|
||||
Actions: chain.Actions{Names: readACLBucketNativeActions},
|
||||
Resources: chain.Resources{Names: []string{
|
||||
fmt.Sprintf(native.ResourceFormatNamespaceContainer, reqInfo.Namespace, cnrIDStr),
|
||||
fmt.Sprintf(native.ResourceFormatNamespaceContainerObjects, reqInfo.Namespace, cnrIDStr),
|
||||
}},
|
||||
})
|
||||
case basicACLPublic:
|
||||
chains[0].Rules = append(chains[0].Rules, chain.Rule{
|
||||
Status: chain.Allow,
|
||||
Actions: chain.Actions{Names: append(readACLBucketS3Actions, writeACLBucketS3Actions...)},
|
||||
Resources: chain.Resources{Names: []string{
|
||||
fmt.Sprintf(s3.ResourceFormatS3Bucket, reqInfo.BucketName),
|
||||
fmt.Sprintf(s3.ResourceFormatS3BucketObjects, reqInfo.BucketName),
|
||||
}},
|
||||
})
|
||||
|
||||
chains[1].Rules = append(chains[1].Rules, chain.Rule{
|
||||
Status: chain.Allow,
|
||||
Actions: chain.Actions{Names: append(readACLBucketNativeActions, writeACLBucketNativeActions...)},
|
||||
Resources: chain.Resources{Names: []string{
|
||||
fmt.Sprintf(native.ResourceFormatNamespaceContainer, reqInfo.Namespace, cnrIDStr),
|
||||
fmt.Sprintf(native.ResourceFormatNamespaceContainerObjects, reqInfo.Namespace, cnrIDStr),
|
||||
}},
|
||||
})
|
||||
default:
|
||||
panic("unknown canned acl") // this should never happen
|
||||
}
|
||||
|
||||
return chains
|
||||
}
|
||||
|
||||
func getBucketCannedChainID(prefix chain.Name, cnrID cid.ID) chain.ID {
|
||||
return chain.ID(string(prefix) + ":bktCanned" + string(cnrID[:]))
|
||||
}
|
||||
|
||||
func (h handler) setPlacementPolicy(prm *layer.CreateBucketParams, namespace, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) error {
|
||||
prm.Policy = h.cfg.DefaultPlacementPolicy(namespace)
|
||||
func (h handler) setPolicy(prm *layer.CreateBucketParams, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) error {
|
||||
prm.Policy = h.cfg.Policy.Default()
|
||||
prm.LocationConstraint = locationConstraint
|
||||
|
||||
if locationConstraint == "" {
|
||||
|
@ -1149,7 +766,7 @@ func (h handler) setPlacementPolicy(prm *layer.CreateBucketParams, namespace, lo
|
|||
}
|
||||
}
|
||||
|
||||
if policy, ok := h.cfg.PlacementPolicy(namespace, locationConstraint); ok {
|
||||
if policy, ok := h.cfg.Policy.Get(locationConstraint); ok {
|
||||
prm.Policy = policy
|
||||
return nil
|
||||
}
|
||||
|
@ -1157,17 +774,9 @@ func (h handler) setPlacementPolicy(prm *layer.CreateBucketParams, namespace, lo
|
|||
return errors.GetAPIError(errors.ErrInvalidLocationConstraint)
|
||||
}
|
||||
|
||||
func isLockEnabled(log *zap.Logger, header http.Header) bool {
|
||||
func isLockEnabled(header http.Header) bool {
|
||||
lockEnabledStr := header.Get(api.AmzBucketObjectLockEnabled)
|
||||
if len(lockEnabledStr) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
lockEnabled, err := strconv.ParseBool(lockEnabledStr)
|
||||
if err != nil {
|
||||
log.Warn(logs.InvalidBucketObjectLockEnabledHeader, zap.String("header", lockEnabledStr), zap.Error(err))
|
||||
}
|
||||
|
||||
lockEnabled, _ := strconv.ParseBool(lockEnabledStr)
|
||||
return lockEnabled
|
||||
}
|
||||
|
||||
|
@ -1205,14 +814,14 @@ func isAlphaNum(char int32) bool {
|
|||
return 'a' <= char && char <= 'z' || '0' <= char && char <= '9'
|
||||
}
|
||||
|
||||
func (h *handler) parseLocationConstraint(r *http.Request) (*createBucketParams, error) {
|
||||
func parseLocationConstraint(r *http.Request) (*createBucketParams, error) {
|
||||
if r.ContentLength == 0 {
|
||||
return new(createBucketParams), nil
|
||||
}
|
||||
|
||||
params := new(createBucketParams)
|
||||
if err := h.cfg.NewXMLDecoder(r.Body).Decode(params); err != nil {
|
||||
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error())
|
||||
if err := xml.NewDecoder(r.Body).Decode(params); err != nil {
|
||||
return nil, errors.GetAPIError(errors.ErrMalformedXML)
|
||||
}
|
||||
return params, nil
|
||||
}
|
||||
|
|
|
@ -1,30 +1,15 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -115,7 +100,7 @@ func TestEmptyPostPolicy(t *testing.T) {
|
|||
},
|
||||
},
|
||||
}
|
||||
reqInfo := &middleware.ReqInfo{}
|
||||
reqInfo := &api.ReqInfo{}
|
||||
metadata := make(map[string]string)
|
||||
|
||||
_, err := checkPostPolicy(r, reqInfo, metadata)
|
||||
|
@ -141,326 +126,3 @@ func TestPutObjectOverrideCopiesNumber(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.Equal(t, "1", objInfo.Headers[layer.AttributeFrostfsCopiesNumber])
|
||||
}
|
||||
|
||||
func TestPutObjectWithNegativeContentLength(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-put", "object-for-put"
|
||||
createTestBucket(tc, bktName)
|
||||
|
||||
content := []byte("content")
|
||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
||||
r.ContentLength = -1
|
||||
tc.Handler().PutObjectHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
w, r = prepareTestRequest(tc, bktName, objName, nil)
|
||||
tc.Handler().HeadObjectHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
require.Equal(t, strconv.Itoa(len(content)), w.Header().Get(api.ContentLength))
|
||||
}
|
||||
|
||||
func TestPutObjectWithStreamBodyError(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-put", "object-for-put"
|
||||
createTestBucket(tc, bktName)
|
||||
|
||||
content := []byte("content")
|
||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
||||
r.Header.Set(api.AmzContentSha256, api.StreamingContentSHA256)
|
||||
r.Header.Set(api.ContentEncoding, api.AwsChunked)
|
||||
tc.Handler().PutObjectHandler(w, r)
|
||||
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrMissingContentLength))
|
||||
|
||||
checkNotFound(t, tc, bktName, objName, emptyVersion)
|
||||
}
|
||||
|
||||
func TestPutObjectWithInvalidContentMD5(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
tc.config.md5Enabled = true
|
||||
|
||||
bktName, objName := "bucket-for-put", "object-for-put"
|
||||
createTestBucket(tc, bktName)
|
||||
|
||||
content := []byte("content")
|
||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString([]byte("invalid")))
|
||||
tc.Handler().PutObjectHandler(w, r)
|
||||
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrInvalidDigest))
|
||||
|
||||
checkNotFound(t, tc, bktName, objName, emptyVersion)
|
||||
}
|
||||
|
||||
func TestPutObjectWithEnabledMD5(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
tc.config.md5Enabled = true
|
||||
|
||||
bktName, objName := "bucket-for-put", "object-for-put"
|
||||
createTestBucket(tc, bktName)
|
||||
|
||||
content := []byte("content")
|
||||
md5Hash := md5.New()
|
||||
md5Hash.Write(content)
|
||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
||||
tc.Handler().PutObjectHandler(w, r)
|
||||
require.Equal(t, data.Quote(hex.EncodeToString(md5Hash.Sum(nil))), w.Header().Get(api.ETag))
|
||||
}
|
||||
|
||||
func TestPutObjectCheckContentSHA256(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-put", "object-for-put"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
hash string
|
||||
content []byte
|
||||
error bool
|
||||
}{
|
||||
{
|
||||
name: "invalid hash value",
|
||||
hash: "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
|
||||
content: []byte("content"),
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "correct hash for empty payload",
|
||||
hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
content: []byte(""),
|
||||
error: false,
|
||||
},
|
||||
{
|
||||
name: "unsigned payload",
|
||||
hash: "UNSIGNED-PAYLOAD",
|
||||
content: []byte("content"),
|
||||
error: false,
|
||||
},
|
||||
{
|
||||
name: "correct hash",
|
||||
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
|
||||
content: []byte("content"),
|
||||
error: false,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w, r := prepareTestPayloadRequest(hc, bktName, objName, bytes.NewReader(tc.content))
|
||||
r.Header.Set("X-Amz-Content-Sha256", tc.hash)
|
||||
hc.Handler().PutObjectHandler(w, r)
|
||||
|
||||
if tc.error {
|
||||
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch))
|
||||
|
||||
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
||||
hc.Handler().GetObjectHandler(w, r)
|
||||
|
||||
assertStatus(t, w, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "examplebucket", "chunkObject.txt"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
w, req, chunk := getChunkedRequest(hc.context, t, bktName, objName)
|
||||
hc.Handler().PutObjectHandler(w, req)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
data := getObjectRange(t, hc, bktName, objName, 0, 66824)
|
||||
for i := range chunk {
|
||||
require.Equal(t, chunk[i], data[i])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutChunkedTestContentEncoding(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "examplebucket", "chunkObject.txt"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
w, req, _ := getChunkedRequest(hc.context, t, bktName, objName)
|
||||
req.Header.Set(api.ContentEncoding, api.AwsChunked+",gzip")
|
||||
|
||||
hc.Handler().PutObjectHandler(w, req)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
resp := headObjectBase(hc, bktName, objName, emptyVersion)
|
||||
require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding))
|
||||
|
||||
w, req, _ = getChunkedRequest(hc.context, t, bktName, objName)
|
||||
req.Header.Set(api.ContentEncoding, "gzip")
|
||||
hc.Handler().PutObjectHandler(w, req)
|
||||
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrInvalidEncodingMethod))
|
||||
|
||||
hc.config.bypassContentEncodingInChunks = true
|
||||
w, req, _ = getChunkedRequest(hc.context, t, bktName, objName)
|
||||
req.Header.Set(api.ContentEncoding, "gzip")
|
||||
hc.Handler().PutObjectHandler(w, req)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
resp = headObjectBase(hc, bktName, objName, emptyVersion)
|
||||
require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding))
|
||||
}
|
||||
|
||||
func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
||||
chunk := make([]byte, 65*1024)
|
||||
for i := range chunk {
|
||||
chunk[i] = 'a'
|
||||
}
|
||||
chunk1 := chunk[:64*1024]
|
||||
chunk2 := chunk[64*1024:]
|
||||
|
||||
AWSAccessKeyID := "AKIAIOSFODNN7EXAMPLE"
|
||||
AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
|
||||
awsCreds := credentials.NewStaticCredentials(AWSAccessKeyID, AWSSecretAccessKey, "")
|
||||
signer := v4.NewSigner(awsCreds)
|
||||
|
||||
reqBody := bytes.NewBufferString("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n")
|
||||
_, err := reqBody.Write(chunk1)
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.WriteString("\r\n400;chunk-signature=0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497\r\n")
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.Write(chunk2)
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.WriteString("\r\n0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n\r\n")
|
||||
require.NoError(t, err)
|
||||
|
||||
req, err := http.NewRequest("PUT", "https://s3.amazonaws.com/"+bktName+"/"+objName, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("content-encoding", "aws-chunked")
|
||||
req.Header.Set("content-length", "66824")
|
||||
req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
|
||||
req.Header.Set("x-amz-decoded-content-length", "66560")
|
||||
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
||||
|
||||
signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = signer.Sign(req, nil, "s3", "us-east-1", signTime)
|
||||
require.NoError(t, err)
|
||||
|
||||
req.Body = io.NopCloser(reqBody)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
||||
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
|
||||
ClientTime: signTime,
|
||||
AuthHeaders: &middleware.AuthHeader{
|
||||
AccessKeyID: AWSAccessKeyID,
|
||||
SignatureV4: "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9",
|
||||
Region: "us-east-1",
|
||||
},
|
||||
AccessBox: &accessbox.Box{
|
||||
Gate: &accessbox.GateData{
|
||||
SecretKey: AWSSecretAccessKey,
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
return w, req, chunk
|
||||
}
|
||||
|
||||
func TestCreateBucket(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName := "bkt-name"
|
||||
|
||||
info := createBucket(hc, bktName)
|
||||
createBucketAssertS3Error(hc, bktName, info.Box, s3errors.ErrBucketAlreadyOwnedByYou)
|
||||
|
||||
box2, _ := createAccessBox(t)
|
||||
createBucketAssertS3Error(hc, bktName, box2, s3errors.ErrBucketAlreadyExists)
|
||||
}
|
||||
|
||||
func TestCreateOldBucketPutVersioning(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
hc.config.aclEnabled = true
|
||||
bktName := "bkt-name"
|
||||
|
||||
info := createBucket(hc, bktName)
|
||||
settings, err := hc.tree.GetSettingsNode(hc.Context(), info.BktInfo)
|
||||
require.NoError(t, err)
|
||||
settings.OwnerKey = nil
|
||||
err = hc.tree.PutSettingsNode(hc.Context(), info.BktInfo, settings)
|
||||
require.NoError(t, err)
|
||||
|
||||
putBucketVersioning(t, hc, bktName, true)
|
||||
}
|
||||
|
||||
func TestCreateNamespacedBucket(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName := "bkt-name"
|
||||
namespace := "yabloko"
|
||||
|
||||
box, _ := createAccessBox(t)
|
||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
reqInfo.Namespace = namespace
|
||||
r = r.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||
hc.Handler().CreateBucketHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
bktInfo, err := hc.Layer().GetBucketInfo(middleware.SetReqInfo(hc.Context(), reqInfo), bktName)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, namespace+".ns", bktInfo.Zone)
|
||||
}
|
||||
|
||||
func TestPutObjectClientCut(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName, objName1, objName2 := "bkt-name", "obj-name1", "obj-name2"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
putObject(hc, bktName, objName1)
|
||||
obj1 := getObjectFromLayer(hc, objName1)[0]
|
||||
require.Empty(t, getObjectAttribute(obj1, "s3-client-cut"))
|
||||
|
||||
hc.layerFeatures.SetClientCut(true)
|
||||
putObject(hc, bktName, objName2)
|
||||
obj2 := getObjectFromLayer(hc, objName2)[0]
|
||||
require.Equal(t, "true", getObjectAttribute(obj2, "s3-client-cut"))
|
||||
}
|
||||
|
||||
func getObjectFromLayer(hc *handlerContext, objName string) []*object.Object {
|
||||
var res []*object.Object
|
||||
for _, o := range hc.tp.Objects() {
|
||||
if objName == getObjectAttribute(o, object.AttributeFilePath) {
|
||||
res = append(res, o)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func getObjectAttribute(obj *object.Object, attrName string) string {
|
||||
for _, attr := range obj.Attributes() {
|
||||
if attr.Key() == attrName {
|
||||
return attr.Value()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestPutObjectWithContentLanguage(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
exceptedContentLanguage := "en"
|
||||
bktName, objName := "bucket-1", "object-1"
|
||||
createTestBucket(tc, bktName)
|
||||
|
||||
w, r := prepareTestRequest(tc, bktName, objName, nil)
|
||||
r.Header.Set(api.ContentLanguage, exceptedContentLanguage)
|
||||
tc.Handler().PutObjectHandler(w, r)
|
||||
|
||||
tc.Handler().HeadObjectHandler(w, r)
|
||||
require.Equal(t, exceptedContentLanguage, w.Header().Get(api.ContentLanguage))
|
||||
}
|
||||
|
|
|
@ -55,19 +55,6 @@ type Bucket struct {
|
|||
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
}
|
||||
|
||||
// PolicyStatus contains status of bucket policy.
|
||||
type PolicyStatus struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PolicyStatus" json:"-"`
|
||||
IsPublic PolicyStatusIsPublic `xml:"IsPublic"`
|
||||
}
|
||||
|
||||
type PolicyStatusIsPublic string
|
||||
|
||||
const (
|
||||
PolicyStatusIsPublicFalse = "FALSE"
|
||||
PolicyStatusIsPublicTrue = "TRUE"
|
||||
)
|
||||
|
||||
// AccessControlPolicy contains ACL.
|
||||
type AccessControlPolicy struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
|
||||
|
@ -117,13 +104,13 @@ type Object struct {
|
|||
Key string
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
ETag string `xml:"ETag,omitempty"`
|
||||
Size uint64
|
||||
Size int64
|
||||
|
||||
// Owner of the object.
|
||||
Owner *Owner `xml:"Owner,omitempty"`
|
||||
|
||||
// Class of storage used to store the object.
|
||||
StorageClass string `xml:"StorageClass"`
|
||||
StorageClass string `xml:"StorageClass,omitempty"`
|
||||
}
|
||||
|
||||
// ObjectVersionResponse container for object version in the response of ListBucketObjectVersionsHandler.
|
||||
|
@ -133,8 +120,8 @@ type ObjectVersionResponse struct {
|
|||
Key string `xml:"Key"`
|
||||
LastModified string `xml:"LastModified"`
|
||||
Owner Owner `xml:"Owner"`
|
||||
Size uint64 `xml:"Size"`
|
||||
StorageClass string `xml:"StorageClass"`
|
||||
Size int64 `xml:"Size"`
|
||||
StorageClass string `xml:"StorageClass,omitempty"` // is empty!!
|
||||
VersionID string `xml:"VersionId"`
|
||||
}
|
||||
|
||||
|
@ -185,6 +172,12 @@ type VersioningConfiguration struct {
|
|||
MfaDelete string `xml:"MfaDelete,omitempty"`
|
||||
}
|
||||
|
||||
// Tagging contains tag set.
|
||||
type Tagging struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"`
|
||||
TagSet []Tag `xml:"TagSet>Tag"`
|
||||
}
|
||||
|
||||
// PostResponse contains result of posting object.
|
||||
type PostResponse struct {
|
||||
Bucket string `xml:"Bucket"`
|
||||
|
@ -192,6 +185,12 @@ type PostResponse struct {
|
|||
ETag string `xml:"Etag"`
|
||||
}
|
||||
|
||||
// Tag is an AWS key-value tag.
|
||||
type Tag struct {
|
||||
Key string
|
||||
Value string
|
||||
}
|
||||
|
||||
// MarshalXML -- StringMap marshals into XML.
|
||||
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
tokens := []xml.Token{start}
|
||||
|
|
|
@ -1,220 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
||||
errs "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
)
|
||||
|
||||
const (
|
||||
chunkSignatureHeader = "chunk-signature="
|
||||
maxChunkSize = 16 << 20
|
||||
)
|
||||
|
||||
type (
|
||||
s3ChunkReader struct {
|
||||
reader *bufio.Reader
|
||||
streamSigner *v4.StreamSigner
|
||||
|
||||
requestTime time.Time
|
||||
buffer []byte
|
||||
offset int
|
||||
err error
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
errGiantChunk = errors.New("chunk too big: choose chunk size <= 16MiB")
|
||||
errMalformedChunkedEncoding = errors.New("malformed chunked encoding")
|
||||
)
|
||||
|
||||
func (c *s3ChunkReader) Close() (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *s3ChunkReader) Read(buf []byte) (num int, err error) {
|
||||
if c.offset > 0 {
|
||||
num = copy(buf, c.buffer[c.offset:])
|
||||
if num == len(buf) {
|
||||
c.offset += num
|
||||
return num, nil
|
||||
}
|
||||
c.offset = 0
|
||||
buf = buf[num:]
|
||||
}
|
||||
|
||||
var size int
|
||||
for {
|
||||
b, err := c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if b == ';' { // separating character
|
||||
break
|
||||
}
|
||||
|
||||
// Manually deserialize the size since AWS specified
|
||||
// the chunk size to be of variable width. In particular,
|
||||
// a size of 16 is encoded as `10` while a size of 64 KB
|
||||
// is `10000`.
|
||||
switch {
|
||||
case b >= '0' && b <= '9':
|
||||
size = size<<4 | int(b-'0')
|
||||
case b >= 'a' && b <= 'f':
|
||||
size = size<<4 | int(b-('a'-10))
|
||||
case b >= 'A' && b <= 'F':
|
||||
size = size<<4 | int(b-('A'-10))
|
||||
default:
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
if size > maxChunkSize {
|
||||
c.err = errGiantChunk
|
||||
return num, c.err
|
||||
}
|
||||
}
|
||||
|
||||
// Now, we read the signature of the following payload and expect:
|
||||
// chunk-signature=" + <signature-as-hex> + "\r\n"
|
||||
//
|
||||
// The signature is 64 bytes long (hex-encoded SHA256 hash) and
|
||||
// starts with a 16 byte header: len("chunk-signature=") + 64 == 80.
|
||||
var signature [80]byte
|
||||
_, err = io.ReadFull(c.reader, signature[:])
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if !bytes.HasPrefix(signature[:], []byte(chunkSignatureHeader)) {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
b, err := c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if b != '\r' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
b, err = c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if b != '\n' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
if cap(c.buffer) < size {
|
||||
c.buffer = make([]byte, size)
|
||||
} else {
|
||||
c.buffer = c.buffer[:size]
|
||||
}
|
||||
|
||||
// Now, we read the payload and compute its SHA-256 hash.
|
||||
_, err = io.ReadFull(c.reader, c.buffer)
|
||||
if err == io.EOF && size != 0 {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil && err != io.EOF {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
b, err = c.reader.ReadByte()
|
||||
if b != '\r' || err != nil {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
b, err = c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if b != '\n' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
// Once we have read the entire chunk successfully, we verify
|
||||
// that the received signature matches our computed signature.
|
||||
|
||||
calculatedSignature, err := c.streamSigner.GetSignature(nil, c.buffer, c.requestTime)
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if string(signature[16:]) != hex.EncodeToString(calculatedSignature) {
|
||||
c.err = errs.GetAPIError(errs.ErrSignatureDoesNotMatch)
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
// If the chunk size is zero we return io.EOF. As specified by AWS,
|
||||
// only the last chunk is zero-sized.
|
||||
if size == 0 {
|
||||
c.err = io.EOF
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
c.offset = copy(buf, c.buffer)
|
||||
num += c.offset
|
||||
return num, err
|
||||
}
|
||||
|
||||
func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, error) {
|
||||
box, err := middleware.GetBoxData(req.Context())
|
||||
if err != nil {
|
||||
return nil, errs.GetAPIError(errs.ErrAuthorizationHeaderMalformed)
|
||||
}
|
||||
|
||||
authHeaders, err := middleware.GetAuthHeaders(req.Context())
|
||||
if err != nil {
|
||||
return nil, errs.GetAPIError(errs.ErrAuthorizationHeaderMalformed)
|
||||
}
|
||||
|
||||
currentCredentials := credentials.NewStaticCredentials(authHeaders.AccessKeyID, box.Gate.SecretKey, "")
|
||||
seed, err := hex.DecodeString(authHeaders.SignatureV4)
|
||||
if err != nil {
|
||||
return nil, errs.GetAPIError(errs.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
|
||||
reqTime, err := middleware.GetClientTime(req.Context())
|
||||
if err != nil {
|
||||
return nil, errs.GetAPIError(errs.ErrMalformedDate)
|
||||
}
|
||||
newStreamSigner := v4.NewStreamSigner(authHeaders.Region, "s3", seed, currentCredentials)
|
||||
|
||||
return &s3ChunkReader{
|
||||
reader: bufio.NewReader(req.Body),
|
||||
streamSigner: newStreamSigner,
|
||||
requestTime: reqTime,
|
||||
buffer: make([]byte, 64*1024),
|
||||
}, nil
|
||||
}
|
|
@ -1,16 +1,17 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -23,10 +24,9 @@ const (
|
|||
)
|
||||
|
||||
func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
tagSet, err := h.readTagSet(reqInfo.Tagging)
|
||||
tagSet, err := readTagSet(r.Body)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not read tag set", reqInfo, err)
|
||||
return
|
||||
|
@ -38,15 +38,15 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
|
|||
return
|
||||
}
|
||||
|
||||
tagPrm := &data.PutObjectTaggingParams{
|
||||
ObjectVersion: &data.ObjectVersion{
|
||||
tagPrm := &layer.PutObjectTaggingParams{
|
||||
ObjectVersion: &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||
},
|
||||
TagSet: tagSet,
|
||||
}
|
||||
nodeVersion, err := h.obj.PutObjectTagging(ctx, tagPrm)
|
||||
nodeVersion, err := h.obj.PutObjectTagging(r.Context(), tagPrm)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not put object tagging", reqInfo, err)
|
||||
return
|
||||
|
@ -63,15 +63,15 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
|
|||
BktInfo: bktInfo,
|
||||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
if err = h.sendNotifications(r.Context(), s); err != nil {
|
||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -85,8 +85,8 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
|
|||
return
|
||||
}
|
||||
|
||||
tagPrm := &data.GetObjectTaggingParams{
|
||||
ObjectVersion: &data.ObjectVersion{
|
||||
tagPrm := &layer.GetObjectTaggingParams{
|
||||
ObjectVersion: &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||
|
@ -102,14 +102,13 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
|
|||
if settings.VersioningEnabled() {
|
||||
w.Header().Set(api.AmzVersionID, versionID)
|
||||
}
|
||||
if err = middleware.EncodeToResponse(w, encodeTagging(tagSet)); err != nil {
|
||||
if err = api.EncodeToResponse(w, encodeTagging(tagSet)); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -117,13 +116,13 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
|
|||
return
|
||||
}
|
||||
|
||||
p := &data.ObjectVersion{
|
||||
p := &layer.ObjectVersion{
|
||||
BktInfo: bktInfo,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||
}
|
||||
|
||||
nodeVersion, err := h.obj.DeleteObjectTagging(ctx, p)
|
||||
nodeVersion, err := h.obj.DeleteObjectTagging(r.Context(), p)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not delete object tagging", reqInfo, err)
|
||||
return
|
||||
|
@ -140,17 +139,17 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
|
|||
BktInfo: bktInfo,
|
||||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
if err = h.sendNotifications(r.Context(), s); err != nil {
|
||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
tagSet, err := h.readTagSet(reqInfo.Tagging)
|
||||
tagSet, err := readTagSet(r.Body)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not read tag set", reqInfo, err)
|
||||
return
|
||||
|
@ -169,7 +168,7 @@ func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request
|
|||
}
|
||||
|
||||
func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -183,14 +182,14 @@ func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request
|
|||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, encodeTagging(tagSet)); err != nil {
|
||||
if err = api.EncodeToResponse(w, encodeTagging(tagSet)); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -205,26 +204,28 @@ func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Requ
|
|||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *handler) readTagSet(tagging *data.Tagging) (map[string]string, error) {
|
||||
func readTagSet(reader io.Reader) (map[string]string, error) {
|
||||
tagging := new(Tagging)
|
||||
if err := xml.NewDecoder(reader).Decode(tagging); err != nil {
|
||||
return nil, errors.GetAPIError(errors.ErrMalformedXML)
|
||||
}
|
||||
|
||||
if err := checkTagSet(tagging.TagSet); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tagSet := make(map[string]string, len(tagging.TagSet))
|
||||
for _, tag := range tagging.TagSet {
|
||||
if _, ok := tagSet[tag.Key]; ok {
|
||||
return nil, errors.GetAPIError(errors.ErrInvalidTagKeyUniqueness)
|
||||
}
|
||||
tagSet[tag.Key] = tag.Value
|
||||
}
|
||||
|
||||
return tagSet, nil
|
||||
}
|
||||
|
||||
func encodeTagging(tagSet map[string]string) *data.Tagging {
|
||||
tagging := &data.Tagging{}
|
||||
func encodeTagging(tagSet map[string]string) *Tagging {
|
||||
tagging := &Tagging{}
|
||||
for k, v := range tagSet {
|
||||
tagging.TagSet = append(tagging.TagSet, data.Tag{Key: k, Value: v})
|
||||
tagging.TagSet = append(tagging.TagSet, Tag{Key: k, Value: v})
|
||||
}
|
||||
sort.Slice(tagging.TagSet, func(i, j int) bool {
|
||||
return tagging.TagSet[i].Key < tagging.TagSet[j].Key
|
||||
|
@ -233,7 +234,7 @@ func encodeTagging(tagSet map[string]string) *data.Tagging {
|
|||
return tagging
|
||||
}
|
||||
|
||||
func checkTagSet(tagSet []data.Tag) error {
|
||||
func checkTagSet(tagSet []Tag) error {
|
||||
if len(tagSet) > maxTags {
|
||||
return errors.GetAPIError(errors.ErrInvalidTagsSizeExceed)
|
||||
}
|
||||
|
@ -247,7 +248,7 @@ func checkTagSet(tagSet []data.Tag) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func checkTag(tag data.Tag) error {
|
||||
func checkTag(tag Tag) error {
|
||||
if len(tag.Key) < 1 || len(tag.Key) > keyTagMaxLength {
|
||||
return errors.GetAPIError(errors.ErrInvalidTagKey)
|
||||
}
|
||||
|
|
|
@ -1,13 +1,9 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -22,23 +18,23 @@ func TestTagsValidity(t *testing.T) {
|
|||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
tag data.Tag
|
||||
tag Tag
|
||||
valid bool
|
||||
}{
|
||||
{tag: data.Tag{}, valid: false},
|
||||
{tag: data.Tag{Key: "", Value: "1"}, valid: false},
|
||||
{tag: data.Tag{Key: "aws:key", Value: "val"}, valid: false},
|
||||
{tag: data.Tag{Key: "key~", Value: "val"}, valid: false},
|
||||
{tag: data.Tag{Key: "key\\", Value: "val"}, valid: false},
|
||||
{tag: data.Tag{Key: "key?", Value: "val"}, valid: false},
|
||||
{tag: data.Tag{Key: sbKey.String() + "b", Value: "val"}, valid: false},
|
||||
{tag: data.Tag{Key: "key", Value: sbValue.String() + "b"}, valid: false},
|
||||
{tag: Tag{}, valid: false},
|
||||
{tag: Tag{Key: "", Value: "1"}, valid: false},
|
||||
{tag: Tag{Key: "aws:key", Value: "val"}, valid: false},
|
||||
{tag: Tag{Key: "key~", Value: "val"}, valid: false},
|
||||
{tag: Tag{Key: "key\\", Value: "val"}, valid: false},
|
||||
{tag: Tag{Key: "key?", Value: "val"}, valid: false},
|
||||
{tag: Tag{Key: sbKey.String() + "b", Value: "val"}, valid: false},
|
||||
{tag: Tag{Key: "key", Value: sbValue.String() + "b"}, valid: false},
|
||||
|
||||
{tag: data.Tag{Key: sbKey.String(), Value: "val"}, valid: true},
|
||||
{tag: data.Tag{Key: "key", Value: sbValue.String()}, valid: true},
|
||||
{tag: data.Tag{Key: "k e y", Value: "v a l"}, valid: true},
|
||||
{tag: data.Tag{Key: "12345", Value: "1234"}, valid: true},
|
||||
{tag: data.Tag{Key: allowedTagChars, Value: allowedTagChars}, valid: true},
|
||||
{tag: Tag{Key: sbKey.String(), Value: "val"}, valid: true},
|
||||
{tag: Tag{Key: "key", Value: sbValue.String()}, valid: true},
|
||||
{tag: Tag{Key: "k e y", Value: "v a l"}, valid: true},
|
||||
{tag: Tag{Key: "12345", Value: "1234"}, valid: true},
|
||||
{tag: Tag{Key: allowedTagChars, Value: allowedTagChars}, valid: true},
|
||||
} {
|
||||
err := checkTag(tc.tag)
|
||||
if tc.valid {
|
||||
|
@ -48,67 +44,3 @@ func TestTagsValidity(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutObjectTaggingCheckUniqueness(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-1", "object-1"
|
||||
createBucketAndObject(hc, bktName, objName)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body *data.Tagging
|
||||
error bool
|
||||
}{
|
||||
{
|
||||
name: "Two tags with unique keys",
|
||||
body: &data.Tagging{
|
||||
TagSet: []data.Tag{
|
||||
{
|
||||
Key: "key-1",
|
||||
Value: "val-1",
|
||||
},
|
||||
{
|
||||
Key: "key-2",
|
||||
Value: "val-2",
|
||||
},
|
||||
},
|
||||
},
|
||||
error: false,
|
||||
},
|
||||
{
|
||||
name: "Two tags with the same keys",
|
||||
body: &data.Tagging{
|
||||
TagSet: []data.Tag{
|
||||
{
|
||||
Key: "key-1",
|
||||
Value: "val-1",
|
||||
},
|
||||
{
|
||||
Key: "key-1",
|
||||
Value: "val-2",
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w, r := prepareTestRequest(hc, bktName, objName, tc.body)
|
||||
middleware.GetReqInfo(r.Context()).Tagging = tc.body
|
||||
hc.Handler().PutObjectTaggingHandler(w, r)
|
||||
if tc.error {
|
||||
assertS3Error(t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidTagKeyUniqueness))
|
||||
return
|
||||
}
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
tagging := getObjectTagging(t, hc, bktName, objName, emptyVersion)
|
||||
require.Len(t, tagging.TagSet, 2)
|
||||
require.Equal(t, "key-1", tagging.TagSet[0].Key)
|
||||
require.Equal(t, "val-1", tagging.TagSet[0].Value)
|
||||
require.Equal(t, "key-2", tagging.TagSet[1].Key)
|
||||
require.Equal(t, "val-2", tagging.TagSet[1].Value)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -3,58 +3,58 @@ package handler
|
|||
import (
|
||||
"net/http"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
)
|
||||
|
||||
func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketAccelerateHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketLoggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
|
|
@ -2,41 +2,23 @@ package handler
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
errorsStd "errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func (h *handler) reqLogger(ctx context.Context) *zap.Logger {
|
||||
reqLogger := middleware.GetReqLog(ctx)
|
||||
if reqLogger != nil {
|
||||
return reqLogger
|
||||
}
|
||||
return h.log
|
||||
}
|
||||
|
||||
func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo *middleware.ReqInfo, err error, additional ...zap.Field) {
|
||||
err = handleDeleteMarker(w, err)
|
||||
if code, wrErr := middleware.WriteErrorResponse(w, reqInfo, transformToS3Error(err)); wrErr != nil {
|
||||
additional = append(additional, zap.NamedError("write_response_error", wrErr))
|
||||
} else {
|
||||
additional = append(additional, zap.Int("status", code))
|
||||
}
|
||||
func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo *api.ReqInfo, err error, additional ...zap.Field) {
|
||||
code := api.WriteErrorResponse(w, reqInfo, transformToS3Error(err))
|
||||
fields := []zap.Field{
|
||||
zap.Int("status", code),
|
||||
zap.String("request_id", reqInfo.RequestID),
|
||||
zap.String("method", reqInfo.API),
|
||||
zap.String("bucket", reqInfo.BucketName),
|
||||
|
@ -44,48 +26,26 @@ func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo
|
|||
zap.String("description", logText),
|
||||
zap.Error(err)}
|
||||
fields = append(fields, additional...)
|
||||
if traceID, err := trace.TraceIDFromHex(reqInfo.TraceID); err == nil && traceID.IsValid() {
|
||||
fields = append(fields, zap.String("trace_id", reqInfo.TraceID))
|
||||
}
|
||||
h.log.Error(logs.RequestFailed, fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
|
||||
}
|
||||
|
||||
func handleDeleteMarker(w http.ResponseWriter, err error) error {
|
||||
var target layer.DeleteMarkerError
|
||||
if !errors.As(err, &target) {
|
||||
return err
|
||||
}
|
||||
|
||||
w.Header().Set(api.AmzDeleteMarker, "true")
|
||||
return fmt.Errorf("%w: %s", s3errors.GetAPIError(target.ErrorCode), err)
|
||||
h.log.Error("call method", fields...)
|
||||
}
|
||||
|
||||
func transformToS3Error(err error) error {
|
||||
err = frosterrors.UnwrapErr(err) // this wouldn't work with errors.Join
|
||||
if _, ok := err.(s3errors.Error); ok {
|
||||
if _, ok := err.(errors.Error); ok {
|
||||
return err
|
||||
}
|
||||
|
||||
if errors.Is(err, layer.ErrAccessDenied) ||
|
||||
errors.Is(err, layer.ErrNodeAccessDenied) {
|
||||
return s3errors.GetAPIError(s3errors.ErrAccessDenied)
|
||||
if errorsStd.Is(err, layer.ErrAccessDenied) ||
|
||||
errorsStd.Is(err, layer.ErrNodeAccessDenied) {
|
||||
return errors.GetAPIError(errors.ErrAccessDenied)
|
||||
}
|
||||
|
||||
if errors.Is(err, layer.ErrGatewayTimeout) {
|
||||
return s3errors.GetAPIError(s3errors.ErrGatewayTimeout)
|
||||
}
|
||||
|
||||
return s3errors.GetAPIError(s3errors.ErrInternalError)
|
||||
return errors.GetAPIError(errors.ErrInternalError)
|
||||
}
|
||||
|
||||
func (h *handler) ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) {
|
||||
return h.obj.GetBucketInfo(ctx, bucket)
|
||||
}
|
||||
|
||||
func (h *handler) ResolveCID(ctx context.Context, bucket string) (cid.ID, error) {
|
||||
return h.obj.ResolveCID(ctx, bucket)
|
||||
}
|
||||
|
||||
func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header ...string) (*data.BucketInfo, error) {
|
||||
bktInfo, err := h.obj.GetBucketInfo(r.Context(), bucket)
|
||||
if err != nil {
|
||||
|
@ -114,26 +74,26 @@ func parseRange(s string) (*layer.RangeParams, error) {
|
|||
prefix := "bytes="
|
||||
|
||||
if !strings.HasPrefix(s, prefix) {
|
||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidRange)
|
||||
return nil, errors.GetAPIError(errors.ErrInvalidRange)
|
||||
}
|
||||
|
||||
s = strings.TrimPrefix(s, prefix)
|
||||
|
||||
valuesStr := strings.Split(s, "-")
|
||||
if len(valuesStr) != 2 {
|
||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidRange)
|
||||
return nil, errors.GetAPIError(errors.ErrInvalidRange)
|
||||
}
|
||||
|
||||
values := make([]uint64, 0, len(valuesStr))
|
||||
for _, v := range valuesStr {
|
||||
num, err := strconv.ParseUint(v, 10, 64)
|
||||
if err != nil {
|
||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidRange)
|
||||
return nil, errors.GetAPIError(errors.ErrInvalidRange)
|
||||
}
|
||||
values = append(values, num)
|
||||
}
|
||||
if values[0] > values[1] {
|
||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidRange)
|
||||
return nil, errors.GetAPIError(errors.ErrInvalidRange)
|
||||
}
|
||||
|
||||
return &layer.RangeParams{
|
||||
|
@ -143,13 +103,13 @@ func parseRange(s string) (*layer.RangeParams, error) {
|
|||
}
|
||||
|
||||
func getSessionTokenSetEACL(ctx context.Context) (*session.Container, error) {
|
||||
boxData, err := middleware.GetBoxData(ctx)
|
||||
boxData, err := layer.GetBoxData(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sessionToken := boxData.Gate.SessionTokenForSetEACL()
|
||||
if sessionToken == nil {
|
||||
return nil, s3errors.GetAPIError(s3errors.ErrAccessDenied)
|
||||
return nil, errors.GetAPIError(errors.ErrAccessDenied)
|
||||
}
|
||||
|
||||
return sessionToken, nil
|
||||
|
|
|
@ -1,64 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestTransformS3Errors(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
expected s3errors.ErrorCode
|
||||
}{
|
||||
{
|
||||
name: "simple std error to internal error",
|
||||
err: errors.New("some error"),
|
||||
expected: s3errors.ErrInternalError,
|
||||
},
|
||||
{
|
||||
name: "layer access denied error to s3 access denied error",
|
||||
err: layer.ErrAccessDenied,
|
||||
expected: s3errors.ErrAccessDenied,
|
||||
},
|
||||
{
|
||||
name: "wrapped layer access denied error to s3 access denied error",
|
||||
err: fmt.Errorf("wrap: %w", layer.ErrAccessDenied),
|
||||
expected: s3errors.ErrAccessDenied,
|
||||
},
|
||||
{
|
||||
name: "layer node access denied error to s3 access denied error",
|
||||
err: layer.ErrNodeAccessDenied,
|
||||
expected: s3errors.ErrAccessDenied,
|
||||
},
|
||||
{
|
||||
name: "layer gateway timeout error to s3 gateway timeout error",
|
||||
err: layer.ErrGatewayTimeout,
|
||||
expected: s3errors.ErrGatewayTimeout,
|
||||
},
|
||||
{
|
||||
name: "s3 error to s3 error",
|
||||
err: s3errors.GetAPIError(s3errors.ErrInvalidPart),
|
||||
expected: s3errors.ErrInvalidPart,
|
||||
},
|
||||
{
|
||||
name: "wrapped s3 error to s3 error",
|
||||
err: fmt.Errorf("wrap: %w", s3errors.GetAPIError(s3errors.ErrInvalidPart)),
|
||||
expected: s3errors.ErrInvalidPart,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := transformToS3Error(tc.err)
|
||||
s3err, ok := err.(s3errors.Error)
|
||||
require.True(t, ok, "error must be s3 error")
|
||||
require.Equalf(t, tc.expected, s3err.ErrCode,
|
||||
"expected: '%s', got: '%s'",
|
||||
s3errors.GetAPIError(tc.expected).Code, s3errors.GetAPIError(s3err.ErrCode).Code)
|
||||
})
|
||||
}
|
||||
}
|
|
@ -1,19 +1,20 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
)
|
||||
|
||||
func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
configuration := new(VersioningConfiguration)
|
||||
if err := h.cfg.NewXMLDecoder(r.Body).Decode(configuration); err != nil {
|
||||
if err := xml.NewDecoder(r.Body).Decode(configuration); err != nil {
|
||||
h.logAndSendError(w, "couldn't decode versioning configuration", reqInfo, errors.GetAPIError(errors.ErrIllegalVersioningConfigurationException))
|
||||
return
|
||||
}
|
||||
|
@ -56,7 +57,7 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
|
|||
|
||||
// GetBucketVersioningHandler implements bucket versioning getter handler.
|
||||
func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := middleware.GetReqInfo(r.Context())
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
|
@ -70,7 +71,7 @@ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
|
|||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, formVersioningConfiguration(settings)); err != nil {
|
||||
if err = api.EncodeToResponse(w, formVersioningConfiguration(settings)); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,5 @@
|
|||
package api
|
||||
|
||||
import "net/http"
|
||||
|
||||
// Standard S3 HTTP request/response constants.
|
||||
const (
|
||||
MetadataPrefix = "X-Amz-Meta-"
|
||||
|
@ -41,13 +39,11 @@ const (
|
|||
IfMatch = "If-Match"
|
||||
IfNoneMatch = "If-None-Match"
|
||||
|
||||
AmzContentSha256 = "X-Amz-Content-Sha256"
|
||||
AmzCopyIfModifiedSince = "X-Amz-Copy-Source-If-Modified-Since"
|
||||
AmzCopyIfUnmodifiedSince = "X-Amz-Copy-Source-If-Unmodified-Since"
|
||||
AmzCopyIfMatch = "X-Amz-Copy-Source-If-Match"
|
||||
AmzCopyIfNoneMatch = "X-Amz-Copy-Source-If-None-Match"
|
||||
AmzACL = "X-Amz-Acl"
|
||||
AmzDecodedContentLength = "X-Amz-Decoded-Content-Length"
|
||||
AmzGrantFullControl = "X-Amz-Grant-Full-Control"
|
||||
AmzGrantRead = "X-Amz-Grant-Read"
|
||||
AmzGrantWrite = "X-Amz-Grant-Write"
|
||||
|
@ -61,17 +57,11 @@ const (
|
|||
AmzObjectAttributes = "X-Amz-Object-Attributes"
|
||||
AmzMaxParts = "X-Amz-Max-Parts"
|
||||
AmzPartNumberMarker = "X-Amz-Part-Number-Marker"
|
||||
AmzStorageClass = "X-Amz-Storage-Class"
|
||||
|
||||
AmzServerSideEncryptionCustomerAlgorithm = "x-amz-server-side-encryption-customer-algorithm"
|
||||
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
|
||||
AmzServerSideEncryptionCustomerKeyMD5 = "x-amz-server-side-encryption-customer-key-MD5"
|
||||
|
||||
AmzCopySourceServerSideEncryptionCustomerAlgorithm = "x-amz-copy-source-server-side-encryption-customer-algorithm"
|
||||
AmzCopySourceServerSideEncryptionCustomerKey = "x-amz-copy-source-server-side-encryption-customer-key"
|
||||
AmzCopySourceServerSideEncryptionCustomerKeyMD5 = "x-amz-copy-source-server-side-encryption-customer-key-MD5"
|
||||
|
||||
OwnerID = "X-Owner-Id"
|
||||
ContainerID = "X-Container-Id"
|
||||
ContainerName = "X-Container-Name"
|
||||
ContainerZone = "X-Container-Zone"
|
||||
|
@ -87,15 +77,9 @@ const (
|
|||
AccessControlRequestMethod = "Access-Control-Request-Method"
|
||||
AccessControlRequestHeaders = "Access-Control-Request-Headers"
|
||||
|
||||
AwsChunked = "aws-chunked"
|
||||
|
||||
Vary = "Vary"
|
||||
|
||||
DefaultLocationConstraint = "default"
|
||||
|
||||
StreamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||
|
||||
DefaultStorageClass = "STANDARD"
|
||||
)
|
||||
|
||||
// S3 request query params.
|
||||
|
@ -121,10 +105,4 @@ var SystemMetadata = map[string]struct{}{
|
|||
ContentType: {},
|
||||
LastModified: {},
|
||||
ETag: {},
|
||||
ContentLanguage: {},
|
||||
}
|
||||
|
||||
func IsSignedStreamingV4(r *http.Request) bool {
|
||||
return r.Header.Get(AmzContentSha256) == StreamingContentSHA256 &&
|
||||
r.Method == http.MethodPut
|
||||
}
|
||||
|
|
|
@ -1,71 +0,0 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
type HostBucketRouter struct {
|
||||
routes map[string]chi.Router
|
||||
bktParam string
|
||||
defaultRouter chi.Router
|
||||
}
|
||||
|
||||
func NewHostBucketRouter(bktParam string) HostBucketRouter {
|
||||
return HostBucketRouter{
|
||||
routes: make(map[string]chi.Router),
|
||||
bktParam: bktParam,
|
||||
}
|
||||
}
|
||||
|
||||
func (hr *HostBucketRouter) Default(router chi.Router) {
|
||||
hr.defaultRouter = router
|
||||
}
|
||||
|
||||
func (hr HostBucketRouter) Map(host string, h chi.Router) {
|
||||
hr.routes[strings.ToLower(host)] = h
|
||||
}
|
||||
|
||||
func (hr HostBucketRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
bucket, domain := getBucketDomain(getHost(r))
|
||||
router, ok := hr.routes[strings.ToLower(domain)]
|
||||
if !ok {
|
||||
router = hr.defaultRouter
|
||||
if router == nil {
|
||||
http.Error(w, http.StatusText(404), 404)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if rctx := chi.RouteContext(r.Context()); rctx != nil && bucket != "" {
|
||||
rctx.URLParams.Add(hr.bktParam, bucket)
|
||||
}
|
||||
|
||||
router.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func getBucketDomain(host string) (bucket string, domain string) {
|
||||
parts := strings.Split(host, ".")
|
||||
if len(parts) > 1 {
|
||||
return parts[0], strings.Join(parts[1:], ".")
|
||||
}
|
||||
return "", host
|
||||
}
|
||||
|
||||
// getHost tries its best to return the request host.
|
||||
// According to section 14.23 of RFC 2616 the Host header
|
||||
// can include the port number if the default value of 80 is not used.
|
||||
func getHost(r *http.Request) string {
|
||||
host := r.Host
|
||||
if r.URL.IsAbs() {
|
||||
host = r.URL.Host
|
||||
}
|
||||
|
||||
if i := strings.Index(host, ":"); i != -1 {
|
||||
host = host[:i]
|
||||
}
|
||||
|
||||
return host
|
||||
}
|
|
@ -1,24 +1,22 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type Cache struct {
|
||||
logger *zap.Logger
|
||||
listsCache *cache.ObjectsListCache
|
||||
sessionListCache *cache.ListSessionCache
|
||||
objCache *cache.ObjectsCache
|
||||
namesCache *cache.ObjectsNameCache
|
||||
bucketCache *cache.BucketCache
|
||||
systemCache *cache.SystemCache
|
||||
accessCache *cache.AccessControlCache
|
||||
logger *zap.Logger
|
||||
listsCache *cache.ObjectsListCache
|
||||
objCache *cache.ObjectsCache
|
||||
namesCache *cache.ObjectsNameCache
|
||||
bucketCache *cache.BucketCache
|
||||
systemCache *cache.SystemCache
|
||||
accessCache *cache.AccessControlCache
|
||||
}
|
||||
|
||||
// CachesConfig contains params for caches.
|
||||
|
@ -26,7 +24,6 @@ type CachesConfig struct {
|
|||
Logger *zap.Logger
|
||||
Objects *cache.Config
|
||||
ObjectsList *cache.Config
|
||||
SessionList *cache.Config
|
||||
Names *cache.Config
|
||||
Buckets *cache.Config
|
||||
System *cache.Config
|
||||
|
@ -39,7 +36,6 @@ func DefaultCachesConfigs(logger *zap.Logger) *CachesConfig {
|
|||
Logger: logger,
|
||||
Objects: cache.DefaultObjectsConfig(logger),
|
||||
ObjectsList: cache.DefaultObjectsListConfig(logger),
|
||||
SessionList: cache.DefaultListSessionConfig(logger),
|
||||
Names: cache.DefaultObjectsNameConfig(logger),
|
||||
Buckets: cache.DefaultBucketConfig(logger),
|
||||
System: cache.DefaultSystemConfig(logger),
|
||||
|
@ -49,33 +45,31 @@ func DefaultCachesConfigs(logger *zap.Logger) *CachesConfig {
|
|||
|
||||
func NewCache(cfg *CachesConfig) *Cache {
|
||||
return &Cache{
|
||||
logger: cfg.Logger,
|
||||
listsCache: cache.NewObjectsListCache(cfg.ObjectsList),
|
||||
sessionListCache: cache.NewListSessionCache(cfg.SessionList),
|
||||
objCache: cache.New(cfg.Objects),
|
||||
namesCache: cache.NewObjectsNameCache(cfg.Names),
|
||||
bucketCache: cache.NewBucketCache(cfg.Buckets),
|
||||
systemCache: cache.NewSystemCache(cfg.System),
|
||||
accessCache: cache.NewAccessControlCache(cfg.AccessControl),
|
||||
logger: cfg.Logger,
|
||||
listsCache: cache.NewObjectsListCache(cfg.ObjectsList),
|
||||
objCache: cache.New(cfg.Objects),
|
||||
namesCache: cache.NewObjectsNameCache(cfg.Names),
|
||||
bucketCache: cache.NewBucketCache(cfg.Buckets),
|
||||
systemCache: cache.NewSystemCache(cfg.System),
|
||||
accessCache: cache.NewAccessControlCache(cfg.AccessControl),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) GetBucket(zone, name string) *data.BucketInfo {
|
||||
return c.bucketCache.Get(zone, name)
|
||||
func (c *Cache) GetBucket(name string) *data.BucketInfo {
|
||||
return c.bucketCache.Get(name)
|
||||
}
|
||||
|
||||
func (c *Cache) PutBucket(bktInfo *data.BucketInfo) {
|
||||
if err := c.bucketCache.Put(bktInfo); err != nil {
|
||||
c.logger.Warn(logs.CouldntPutBucketInfoIntoCache,
|
||||
zap.String("zone", bktInfo.Zone),
|
||||
c.logger.Warn("couldn't put bucket info into cache",
|
||||
zap.String("bucket name", bktInfo.Name),
|
||||
zap.Stringer("bucket cid", bktInfo.CID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) DeleteBucket(bktInfo *data.BucketInfo) {
|
||||
c.bucketCache.Delete(bktInfo)
|
||||
func (c *Cache) DeleteBucket(name string) {
|
||||
c.bucketCache.Delete(name)
|
||||
}
|
||||
|
||||
func (c *Cache) CleanListCacheEntriesContainingObject(objectName string, cnrID cid.ID) {
|
||||
|
@ -110,13 +104,13 @@ func (c *Cache) GetLastObject(owner user.ID, bktName, objName string) *data.Exte
|
|||
|
||||
func (c *Cache) PutObject(owner user.ID, extObjInfo *data.ExtendedObjectInfo) {
|
||||
if err := c.objCache.PutObject(extObjInfo); err != nil {
|
||||
c.logger.Warn(logs.CouldntAddObjectToCache, zap.Error(err),
|
||||
c.logger.Warn("couldn't add object to cache", zap.Error(err),
|
||||
zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket),
|
||||
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -124,7 +118,7 @@ func (c *Cache) PutObjectWithName(owner user.ID, extObjInfo *data.ExtendedObject
|
|||
c.PutObject(owner, extObjInfo)
|
||||
|
||||
if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil {
|
||||
c.logger.Warn(logs.CouldntPutObjAddressToNameCache,
|
||||
c.logger.Warn("couldn't put obj address to name cache",
|
||||
zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()),
|
||||
zap.Error(err))
|
||||
}
|
||||
|
@ -140,37 +134,14 @@ func (c *Cache) GetList(owner user.ID, key cache.ObjectsListKey) []*data.NodeVer
|
|||
|
||||
func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) {
|
||||
if err := c.listsCache.PutVersions(key, list); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err))
|
||||
c.logger.Warn("couldn't cache list of objects", zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) GetListSession(owner user.ID, key cache.ListSessionKey) *data.ListSession {
|
||||
if !c.accessCache.Get(owner, key.String()) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return c.sessionListCache.GetListSession(key)
|
||||
}
|
||||
|
||||
func (c *Cache) PutListSession(owner user.ID, key cache.ListSessionKey, session *data.ListSession) {
|
||||
if err := c.sessionListCache.PutListSession(key, session); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheListSession, zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) DeleteListSession(owner user.ID, key cache.ListSessionKey) {
|
||||
c.sessionListCache.DeleteListSession(key)
|
||||
c.accessCache.Delete(owner, key.String())
|
||||
}
|
||||
|
||||
func (c *Cache) GetTagging(owner user.ID, key string) map[string]string {
|
||||
if !c.accessCache.Get(owner, key) {
|
||||
return nil
|
||||
|
@ -181,11 +152,11 @@ func (c *Cache) GetTagging(owner user.ID, key string) map[string]string {
|
|||
|
||||
func (c *Cache) PutTagging(owner user.ID, key string, tags map[string]string) {
|
||||
if err := c.systemCache.PutTagging(key, tags); err != nil {
|
||||
c.logger.Error(logs.CouldntCacheTags, zap.Error(err))
|
||||
c.logger.Error("couldn't cache tags", zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -203,11 +174,11 @@ func (c *Cache) GetLockInfo(owner user.ID, key string) *data.LockInfo {
|
|||
|
||||
func (c *Cache) PutLockInfo(owner user.ID, key string, lockInfo *data.LockInfo) {
|
||||
if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil {
|
||||
c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err))
|
||||
c.logger.Error("couldn't cache lock info", zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -224,11 +195,11 @@ func (c *Cache) GetSettings(owner user.ID, bktInfo *data.BucketInfo) *data.Bucke
|
|||
func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
|
||||
key := bktInfo.Name + bktInfo.SettingsObjectName()
|
||||
if err := c.systemCache.PutSettings(key, settings); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||
c.logger.Warn("couldn't cache bucket settings", zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -246,11 +217,11 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
|
|||
key := bkt.Name + bkt.CORSObjectName()
|
||||
|
||||
if err := c.systemCache.PutCORS(key, cors); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err))
|
||||
c.logger.Warn("couldn't cache cors", zap.String("bucket", bkt.Name), zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -271,10 +242,10 @@ func (c *Cache) GetNotificationConfiguration(owner user.ID, bktInfo *data.Bucket
|
|||
func (c *Cache) PutNotificationConfiguration(owner user.ID, bktInfo *data.BucketInfo, configuration *data.NotificationConfiguration) {
|
||||
key := bktInfo.Name + bktInfo.NotificationConfigurationObjectName()
|
||||
if err := c.systemCache.PutNotificationConfiguration(key, configuration); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheNotificationConfiguration, zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||
c.logger.Warn("couldn't cache notification configuration", zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,41 +2,40 @@ package layer
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
errorsStd "errors"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
)
|
||||
|
||||
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
|
||||
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
||||
var err error
|
||||
owner := n.BearerOwner(ctx)
|
||||
owner := n.Owner(ctx)
|
||||
|
||||
tags := n.cache.GetTagging(owner, objectTaggingCacheKey(objVersion))
|
||||
lockInfo := n.cache.GetLockInfo(owner, lockObjectKey(objVersion))
|
||||
|
||||
if tags != nil && lockInfo != nil {
|
||||
return tags, *lockInfo, nil
|
||||
return tags, lockInfo, nil
|
||||
}
|
||||
|
||||
if nodeVersion == nil {
|
||||
nodeVersion, err = n.getNodeVersion(ctx, objVersion)
|
||||
if err != nil {
|
||||
return nil, data.LockInfo{}, err
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
tags, lockInfo, err = n.treeService.GetObjectTaggingAndLock(ctx, objVersion.BktInfo, nodeVersion)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
return nil, data.LockInfo{}, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
|
||||
if errorsStd.Is(err, ErrNodeNotFound) {
|
||||
return nil, nil, errors.GetAPIError(errors.ErrNoSuchKey)
|
||||
}
|
||||
return nil, data.LockInfo{}, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
n.cache.PutTagging(owner, objectTaggingCacheKey(objVersion), tags)
|
||||
n.cache.PutLockInfo(owner, lockObjectKey(objVersion), lockInfo)
|
||||
|
||||
return tags, *lockInfo, nil
|
||||
return tags, lockInfo, nil
|
||||
}
|
||||
|
|
|
@ -5,17 +5,15 @@ import (
|
|||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
v2container "github.com/TrueCloudLab/frostfs-api-go/v2/container"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/client"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -32,24 +30,24 @@ const (
|
|||
AttributeLockEnabled = "LockEnabled"
|
||||
)
|
||||
|
||||
func (n *layer) containerInfo(ctx context.Context, prm PrmContainer) (*data.BucketInfo, error) {
|
||||
func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketInfo, error) {
|
||||
var (
|
||||
err error
|
||||
res *container.Container
|
||||
log = n.reqLogger(ctx).With(zap.Stringer("cid", prm.ContainerID))
|
||||
rid = api.GetRequestID(ctx)
|
||||
log = n.log.With(zap.Stringer("cid", idCnr), zap.String("request_id", rid))
|
||||
|
||||
info = &data.BucketInfo{
|
||||
CID: prm.ContainerID,
|
||||
Name: prm.ContainerID.EncodeToString(),
|
||||
CID: idCnr,
|
||||
Name: idCnr.EncodeToString(),
|
||||
}
|
||||
|
||||
reqInfo = middleware.GetReqInfo(ctx)
|
||||
)
|
||||
|
||||
res, err = n.frostFS.Container(ctx, prm)
|
||||
res, err = n.frostFS.Container(ctx, idCnr)
|
||||
if err != nil {
|
||||
log.Error("could not fetch container", zap.Error(err))
|
||||
|
||||
if client.IsErrContainerNotFound(err) {
|
||||
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchBucket), err.Error())
|
||||
return nil, errors.GetAPIError(errors.ErrNoSuchBucket)
|
||||
}
|
||||
return nil, fmt.Errorf("get frostfs container: %w", err)
|
||||
}
|
||||
|
@ -63,53 +61,45 @@ func (n *layer) containerInfo(ctx context.Context, prm PrmContainer) (*data.Buck
|
|||
}
|
||||
info.Created = container.CreatedAt(cnr)
|
||||
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
|
||||
info.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(cnr)
|
||||
info.APEEnabled = cnr.BasicACL().Bits() == 0
|
||||
|
||||
attrLockEnabled := cnr.Attribute(AttributeLockEnabled)
|
||||
if len(attrLockEnabled) > 0 {
|
||||
info.ObjectLockEnabled, err = strconv.ParseBool(attrLockEnabled)
|
||||
if err != nil {
|
||||
log.Error(logs.CouldNotParseContainerObjectLockEnabledAttribute,
|
||||
log.Error("could not parse container object lock enabled attribute",
|
||||
zap.String("lock_enabled", attrLockEnabled),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
zone, _ := n.features.FormContainerZone(reqInfo.Namespace)
|
||||
if zone != info.Zone {
|
||||
return nil, fmt.Errorf("ns '%s' and zone '%s' are mismatched for container '%s'", zone, info.Zone, prm.ContainerID)
|
||||
}
|
||||
|
||||
n.cache.PutBucket(info)
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||
stoken := n.SessionTokenForRead(ctx)
|
||||
|
||||
prm := PrmUserContainers{
|
||||
UserID: n.BearerOwner(ctx),
|
||||
SessionToken: stoken,
|
||||
}
|
||||
|
||||
res, err := n.frostFS.UserContainers(ctx, prm)
|
||||
var (
|
||||
err error
|
||||
own = n.Owner(ctx)
|
||||
res []cid.ID
|
||||
rid = api.GetRequestID(ctx)
|
||||
)
|
||||
res, err = n.frostFS.UserContainers(ctx, own)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err))
|
||||
n.log.Error("could not list user containers",
|
||||
zap.String("request_id", rid),
|
||||
zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
list := make([]*data.BucketInfo, 0, len(res))
|
||||
for i := range res {
|
||||
getPrm := PrmContainer{
|
||||
ContainerID: res[i],
|
||||
SessionToken: stoken,
|
||||
}
|
||||
info, err := n.containerInfo(ctx, getPrm)
|
||||
info, err := n.containerInfo(ctx, res[i])
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err))
|
||||
n.log.Error("could not fetch container info",
|
||||
zap.String("request_id", rid),
|
||||
zap.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
|
@ -120,25 +110,24 @@ func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
|||
}
|
||||
|
||||
func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) {
|
||||
ownerID := n.Owner(ctx)
|
||||
if p.LocationConstraint == "" {
|
||||
p.LocationConstraint = api.DefaultLocationConstraint // s3tests_boto3.functional.test_s3:test_bucket_get_location
|
||||
}
|
||||
|
||||
zone, _ := n.features.FormContainerZone(p.Namespace)
|
||||
|
||||
bktInfo := &data.BucketInfo{
|
||||
Name: p.Name,
|
||||
Zone: zone,
|
||||
Owner: n.BearerOwner(ctx),
|
||||
Zone: v2container.SysAttributeZoneDefault,
|
||||
Owner: ownerID,
|
||||
Created: TimeNow(ctx),
|
||||
LocationConstraint: p.LocationConstraint,
|
||||
ObjectLockEnabled: p.ObjectLockEnabled,
|
||||
APEEnabled: p.APEEnabled,
|
||||
}
|
||||
|
||||
attributes := [][2]string{
|
||||
{attributeLocationConstraint, p.LocationConstraint},
|
||||
}
|
||||
var attributes [][2]string
|
||||
|
||||
attributes = append(attributes, [2]string{
|
||||
attributeLocationConstraint, p.LocationConstraint,
|
||||
})
|
||||
|
||||
if p.ObjectLockEnabled {
|
||||
attributes = append(attributes, [2]string{
|
||||
|
@ -146,27 +135,23 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
|||
})
|
||||
}
|
||||
|
||||
basicACL := acl.PublicRWExtended
|
||||
if p.APEEnabled {
|
||||
basicACL = 0
|
||||
}
|
||||
|
||||
res, err := n.frostFS.CreateContainer(ctx, PrmContainerCreate{
|
||||
idCnr, err := n.frostFS.CreateContainer(ctx, PrmContainerCreate{
|
||||
Creator: bktInfo.Owner,
|
||||
Policy: p.Policy,
|
||||
Name: p.Name,
|
||||
Zone: zone,
|
||||
SessionToken: p.SessionContainerCreation,
|
||||
CreationTime: bktInfo.Created,
|
||||
AdditionalAttributes: attributes,
|
||||
BasicACL: basicACL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create container: %w", err)
|
||||
}
|
||||
|
||||
bktInfo.CID = res.ContainerID
|
||||
bktInfo.HomomorphicHashDisabled = res.HomomorphicHashDisabled
|
||||
bktInfo.CID = idCnr
|
||||
|
||||
if err = n.setContainerEACLTable(ctx, bktInfo.CID, p.EACL, p.SessionEACL); err != nil {
|
||||
return nil, fmt.Errorf("set container eacl: %w", err)
|
||||
}
|
||||
|
||||
n.cache.PutBucket(bktInfo)
|
||||
|
||||
|
@ -179,10 +164,6 @@ func (n *layer) setContainerEACLTable(ctx context.Context, idCnr cid.ID, table *
|
|||
return n.frostFS.SetContainerEACL(ctx, *table, sessionToken)
|
||||
}
|
||||
|
||||
func (n *layer) GetContainerEACL(ctx context.Context, cnrID cid.ID) (*eacl.Table, error) {
|
||||
prm := PrmContainerEACL{
|
||||
ContainerID: cnrID,
|
||||
SessionToken: n.SessionTokenForRead(ctx),
|
||||
}
|
||||
return n.frostFS.ContainerEACL(ctx, prm)
|
||||
func (n *layer) GetContainerEACL(ctx context.Context, idCnr cid.ID) (*eacl.Table, error) {
|
||||
return n.frostFS.ContainerEACL(ctx, idCnr)
|
||||
}
|
||||
|
|
|
@ -3,13 +3,13 @@ package layer
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
errorsStd "errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -24,7 +24,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
|||
cors = &data.CORSConfiguration{}
|
||||
)
|
||||
|
||||
if err := p.NewDecoder(tee).Decode(cors); err != nil {
|
||||
if err := xml.NewDecoder(tee).Decode(cors); err != nil {
|
||||
return fmt.Errorf("xml decode cors: %w", err)
|
||||
}
|
||||
|
||||
|
@ -38,13 +38,14 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
|||
|
||||
prm := PrmObjectCreate{
|
||||
Container: p.BktInfo.CID,
|
||||
Payload: &buf,
|
||||
Creator: p.BktInfo.Owner,
|
||||
Payload: p.Reader,
|
||||
Filepath: p.BktInfo.CORSObjectName(),
|
||||
CreationTime: TimeNow(ctx),
|
||||
CopiesNumber: p.CopiesNumbers,
|
||||
CopiesNumber: p.CopiesNumber,
|
||||
}
|
||||
|
||||
_, objID, _, _, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
||||
objID, _, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("put system object: %w", err)
|
||||
}
|
||||
|
@ -57,13 +58,14 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
|||
|
||||
if !objIDToDeleteNotFound {
|
||||
if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
|
||||
n.log.Error("couldn't delete cors object", zap.Error(err),
|
||||
zap.String("cnrID", p.BktInfo.CID.EncodeToString()),
|
||||
zap.String("bucket name", p.BktInfo.Name),
|
||||
zap.String("objID", objIDToDelete.EncodeToString()))
|
||||
}
|
||||
}
|
||||
|
||||
n.cache.PutCORS(n.BearerOwner(ctx), p.BktInfo, cors)
|
||||
n.cache.PutCORS(n.Owner(ctx), p.BktInfo, cors)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -72,7 +74,7 @@ func (n *layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*d
|
|||
cors, err := n.getCORS(ctx, bktInfo)
|
||||
if err != nil {
|
||||
if errorsStd.Is(err, ErrNodeNotFound) {
|
||||
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchCORSConfiguration), err.Error())
|
||||
return nil, errors.GetAPIError(errors.ErrNoSuchCORSConfiguration)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -10,7 +10,6 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
|
@ -101,11 +100,8 @@ func (p Params) HMAC() ([]byte, []byte, error) {
|
|||
|
||||
// MatchObjectEncryption checks if encryption params are valid for provided object.
|
||||
func (p Params) MatchObjectEncryption(encInfo ObjectEncryption) error {
|
||||
if p.Enabled() && !encInfo.Enabled {
|
||||
return errors.GetAPIError(errors.ErrInvalidEncryptionParameters)
|
||||
}
|
||||
if !p.Enabled() && encInfo.Enabled {
|
||||
return errors.GetAPIError(errors.ErrSSEEncryptedObject)
|
||||
if p.Enabled() != encInfo.Enabled {
|
||||
return errorsStd.New("invalid encryption view")
|
||||
}
|
||||
|
||||
if !encInfo.Enabled {
|
||||
|
@ -126,7 +122,7 @@ func (p Params) MatchObjectEncryption(encInfo ObjectEncryption) error {
|
|||
mac.Write(hmacSalt)
|
||||
expectedHmacKey := mac.Sum(nil)
|
||||
if !bytes.Equal(expectedHmacKey, hmacKey) {
|
||||
return errors.GetAPIError(errors.ErrInvalidSSECustomerParameters)
|
||||
return errorsStd.New("mismatched hmac key")
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
@ -7,16 +7,16 @@ import (
|
|||
"io"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/container"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
||||
)
|
||||
|
||||
// PrmContainerCreate groups parameters of FrostFS.CreateContainer operation.
|
||||
|
@ -30,9 +30,6 @@ type PrmContainerCreate struct {
|
|||
// Name for the container.
|
||||
Name string
|
||||
|
||||
// Zone for container registration.
|
||||
Zone string
|
||||
|
||||
// CreationTime value for Timestamp attribute
|
||||
CreationTime time.Time
|
||||
|
||||
|
@ -46,39 +43,6 @@ type PrmContainerCreate struct {
|
|||
AdditionalAttributes [][2]string
|
||||
}
|
||||
|
||||
// PrmContainer groups parameters of FrostFS.Container operation.
|
||||
type PrmContainer struct {
|
||||
// Container identifier.
|
||||
ContainerID cid.ID
|
||||
|
||||
// Token of the container's creation session. Nil means session absence.
|
||||
SessionToken *session.Container
|
||||
}
|
||||
|
||||
// PrmUserContainers groups parameters of FrostFS.UserContainers operation.
|
||||
type PrmUserContainers struct {
|
||||
// User identifier.
|
||||
UserID user.ID
|
||||
|
||||
// Token of the container's creation session. Nil means session absence.
|
||||
SessionToken *session.Container
|
||||
}
|
||||
|
||||
// PrmContainerEACL groups parameters of FrostFS.ContainerEACL operation.
|
||||
type PrmContainerEACL struct {
|
||||
// Container identifier.
|
||||
ContainerID cid.ID
|
||||
|
||||
// Token of the container's creation session. Nil means session absence.
|
||||
SessionToken *session.Container
|
||||
}
|
||||
|
||||
// ContainerCreateResult is a result parameter of FrostFS.CreateContainer operation.
|
||||
type ContainerCreateResult struct {
|
||||
ContainerID cid.ID
|
||||
HomomorphicHashDisabled bool
|
||||
}
|
||||
|
||||
// PrmAuth groups authentication parameters for the FrostFS operation.
|
||||
type PrmAuth struct {
|
||||
// Bearer token to be used for the operation. Overlaps PrivateKey. Optional.
|
||||
|
@ -127,6 +91,9 @@ type PrmObjectCreate struct {
|
|||
// Container to store the object.
|
||||
Container cid.ID
|
||||
|
||||
// FrostFS identifier of the object creator.
|
||||
Creator user.ID
|
||||
|
||||
// Key-value object attributes.
|
||||
Attributes [][2]string
|
||||
|
||||
|
@ -146,16 +113,7 @@ type PrmObjectCreate struct {
|
|||
Payload io.Reader
|
||||
|
||||
// Number of object copies that is enough to consider put successful.
|
||||
CopiesNumber []uint32
|
||||
|
||||
// Enables client side object preparing.
|
||||
ClientCut bool
|
||||
|
||||
// Disables using Tillich-Zémor hash for payload.
|
||||
WithoutHomomorphicHash bool
|
||||
|
||||
// Sets max buffer size to read payload.
|
||||
BufferMaxSize uint64
|
||||
CopiesNumber uint32
|
||||
}
|
||||
|
||||
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
||||
|
@ -170,29 +128,8 @@ type PrmObjectDelete struct {
|
|||
Object oid.ID
|
||||
}
|
||||
|
||||
// PrmObjectSearch groups parameters of FrostFS.sear SearchObjects operation.
|
||||
type PrmObjectSearch struct {
|
||||
// Authentication parameters.
|
||||
PrmAuth
|
||||
|
||||
// Container to select the objects from.
|
||||
Container cid.ID
|
||||
|
||||
// Key-value object attribute which should be
|
||||
// presented in selected objects. Optional, empty key means any.
|
||||
ExactAttribute [2]string
|
||||
|
||||
// File prefix of the selected objects. Optional, empty value means any.
|
||||
FilePrefix string
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrAccessDenied is returned from FrostFS in case of access violation.
|
||||
ErrAccessDenied = errors.New("access denied")
|
||||
|
||||
// ErrGatewayTimeout is returned from FrostFS in case of timeout, deadline exceeded etc.
|
||||
ErrGatewayTimeout = errors.New("gateway timeout")
|
||||
)
|
||||
// ErrAccessDenied is returned from FrostFS in case of access violation.
|
||||
var ErrAccessDenied = errors.New("access denied")
|
||||
|
||||
// FrostFS represents virtual connection to FrostFS network.
|
||||
type FrostFS interface {
|
||||
|
@ -200,21 +137,23 @@ type FrostFS interface {
|
|||
// It sets 'Timestamp' attribute to the current time.
|
||||
// It returns the ID of the saved container.
|
||||
//
|
||||
// Created container is public with enabled ACL extension.
|
||||
//
|
||||
// It returns exactly one non-zero value. It returns any error encountered which
|
||||
// prevented the container from being created.
|
||||
CreateContainer(context.Context, PrmContainerCreate) (*ContainerCreateResult, error)
|
||||
CreateContainer(context.Context, PrmContainerCreate) (cid.ID, error)
|
||||
|
||||
// Container reads a container from FrostFS by ID.
|
||||
//
|
||||
// It returns exactly one non-nil value. It returns any error encountered which
|
||||
// prevented the container from being read.
|
||||
Container(context.Context, PrmContainer) (*container.Container, error)
|
||||
Container(context.Context, cid.ID) (*container.Container, error)
|
||||
|
||||
// UserContainers reads a list of the containers owned by the specified user.
|
||||
//
|
||||
// It returns exactly one non-nil value. It returns any error encountered which
|
||||
// prevented the containers from being listed.
|
||||
UserContainers(context.Context, PrmUserContainers) ([]cid.ID, error)
|
||||
UserContainers(context.Context, user.ID) ([]cid.ID, error)
|
||||
|
||||
// SetContainerEACL saves the eACL table of the container in FrostFS. The
|
||||
// extended ACL is modified within session if session token is not nil.
|
||||
|
@ -226,7 +165,7 @@ type FrostFS interface {
|
|||
//
|
||||
// It returns exactly one non-nil value. It returns any error encountered which
|
||||
// prevented the eACL from being read.
|
||||
ContainerEACL(context.Context, PrmContainerEACL) (*eacl.Table, error)
|
||||
ContainerEACL(context.Context, cid.ID) (*eacl.Table, error)
|
||||
|
||||
// DeleteContainer marks the container to be removed from FrostFS by ID.
|
||||
// Request is sent within session if the session token is specified.
|
||||
|
@ -271,15 +210,6 @@ type FrostFS interface {
|
|||
// It returns any error encountered which prevented the removal request from being sent.
|
||||
DeleteObject(context.Context, PrmObjectDelete) error
|
||||
|
||||
// SearchObjects performs object search from the NeoFS container according
|
||||
// to the specified parameters. It searches user's objects only.
|
||||
//
|
||||
// It returns ErrAccessDenied on selection access violation.
|
||||
//
|
||||
// It returns exactly one non-nil value. It returns any error encountered which
|
||||
// prevented the objects from being selected.
|
||||
SearchObjects(context.Context, PrmObjectSearch) ([]oid.ID, error)
|
||||
|
||||
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided now and future time.
|
||||
// Note:
|
||||
// * future time must be after the now
|
||||
|
|
|
@ -1,497 +0,0 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
|
||||
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
||||
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
)
|
||||
|
||||
type FeatureSettingsMock struct {
|
||||
clientCut bool
|
||||
md5Enabled bool
|
||||
}
|
||||
|
||||
func (k *FeatureSettingsMock) BufferMaxSizeForPut() uint64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (k *FeatureSettingsMock) ClientCut() bool {
|
||||
return k.clientCut
|
||||
}
|
||||
|
||||
func (k *FeatureSettingsMock) SetClientCut(clientCut bool) {
|
||||
k.clientCut = clientCut
|
||||
}
|
||||
|
||||
func (k *FeatureSettingsMock) MD5Enabled() bool {
|
||||
return k.md5Enabled
|
||||
}
|
||||
|
||||
func (k *FeatureSettingsMock) SetMD5Enabled(md5Enabled bool) {
|
||||
k.md5Enabled = md5Enabled
|
||||
}
|
||||
|
||||
func (k *FeatureSettingsMock) FormContainerZone(ns string) (zone string, isDefault bool) {
|
||||
if ns == "" {
|
||||
return v2container.SysAttributeZoneDefault, true
|
||||
}
|
||||
|
||||
return ns + ".ns", false
|
||||
}
|
||||
|
||||
type TestFrostFS struct {
|
||||
FrostFS
|
||||
|
||||
objects map[string]*object.Object
|
||||
objectErrors map[string]error
|
||||
objectPutErrors map[string]error
|
||||
containers map[string]*container.Container
|
||||
eaclTables map[string]*eacl.Table
|
||||
currentEpoch uint64
|
||||
key *keys.PrivateKey
|
||||
}
|
||||
|
||||
func NewTestFrostFS(key *keys.PrivateKey) *TestFrostFS {
|
||||
return &TestFrostFS{
|
||||
objects: make(map[string]*object.Object),
|
||||
objectErrors: make(map[string]error),
|
||||
objectPutErrors: make(map[string]error),
|
||||
containers: make(map[string]*container.Container),
|
||||
eaclTables: make(map[string]*eacl.Table),
|
||||
key: key,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) CurrentEpoch() uint64 {
|
||||
return t.currentEpoch
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) SetObjectError(addr oid.Address, err error) {
|
||||
if err == nil {
|
||||
delete(t.objectErrors, addr.EncodeToString())
|
||||
} else {
|
||||
t.objectErrors[addr.EncodeToString()] = err
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) SetObjectPutError(fileName string, err error) {
|
||||
if err == nil {
|
||||
delete(t.objectPutErrors, fileName)
|
||||
} else {
|
||||
t.objectPutErrors[fileName] = err
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) Objects() []*object.Object {
|
||||
res := make([]*object.Object, 0, len(t.objects))
|
||||
|
||||
for _, obj := range t.objects {
|
||||
res = append(res, obj)
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) ObjectExists(objID oid.ID) bool {
|
||||
for _, obj := range t.objects {
|
||||
if id, _ := obj.ID(); id.Equals(objID) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) AddObject(key string, obj *object.Object) {
|
||||
t.objects[key] = obj
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) ContainerID(name string) (cid.ID, error) {
|
||||
for id, cnr := range t.containers {
|
||||
if container.Name(*cnr) == name {
|
||||
var cnrID cid.ID
|
||||
return cnrID, cnrID.DecodeString(id)
|
||||
}
|
||||
}
|
||||
return cid.ID{}, fmt.Errorf("not found")
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) SetContainer(cnrID cid.ID, cnr *container.Container) {
|
||||
t.containers[cnrID.EncodeToString()] = cnr
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (*ContainerCreateResult, error) {
|
||||
var cnr container.Container
|
||||
cnr.Init()
|
||||
cnr.SetOwner(prm.Creator)
|
||||
cnr.SetPlacementPolicy(prm.Policy)
|
||||
cnr.SetBasicACL(prm.BasicACL)
|
||||
|
||||
creationTime := prm.CreationTime
|
||||
if creationTime.IsZero() {
|
||||
creationTime = time.Now()
|
||||
}
|
||||
container.SetCreationTime(&cnr, creationTime)
|
||||
|
||||
if prm.Name != "" {
|
||||
var d container.Domain
|
||||
d.SetName(prm.Name)
|
||||
d.SetZone(prm.Zone)
|
||||
|
||||
container.WriteDomain(&cnr, d)
|
||||
container.SetName(&cnr, prm.Name)
|
||||
}
|
||||
|
||||
for i := range prm.AdditionalAttributes {
|
||||
cnr.SetAttribute(prm.AdditionalAttributes[i][0], prm.AdditionalAttributes[i][1])
|
||||
}
|
||||
|
||||
b := make([]byte, 32)
|
||||
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var id cid.ID
|
||||
id.SetSHA256(sha256.Sum256(b))
|
||||
t.containers[id.EncodeToString()] = &cnr
|
||||
|
||||
return &ContainerCreateResult{ContainerID: id}, nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *session.Container) error {
|
||||
delete(t.containers, cnrID.EncodeToString())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) Container(_ context.Context, prm PrmContainer) (*container.Container, error) {
|
||||
for k, v := range t.containers {
|
||||
if k == prm.ContainerID.EncodeToString() {
|
||||
return v, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("container not found %s", prm.ContainerID)
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) UserContainers(context.Context, PrmUserContainers) ([]cid.ID, error) {
|
||||
var res []cid.ID
|
||||
for k := range t.containers {
|
||||
var idCnr cid.ID
|
||||
if err := idCnr.DecodeString(k); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res = append(res, idCnr)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectPart, error) {
|
||||
var addr oid.Address
|
||||
addr.SetContainer(prm.Container)
|
||||
addr.SetObject(prm.Object)
|
||||
|
||||
sAddr := addr.EncodeToString()
|
||||
|
||||
if err := t.objectErrors[sAddr]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if obj, ok := t.objects[sAddr]; ok {
|
||||
owner := getBearerOwner(ctx)
|
||||
if !t.checkAccess(prm.Container, owner, eacl.OperationGet, obj) {
|
||||
return nil, ErrAccessDenied
|
||||
}
|
||||
|
||||
payload := obj.Payload()
|
||||
|
||||
if prm.PayloadRange[0]+prm.PayloadRange[1] > 0 {
|
||||
off := prm.PayloadRange[0]
|
||||
payload = payload[off : off+prm.PayloadRange[1]]
|
||||
}
|
||||
|
||||
return &ObjectPart{
|
||||
Head: obj,
|
||||
Payload: io.NopCloser(bytes.NewReader(payload)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, addr)
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID, error) {
|
||||
b := make([]byte, 32)
|
||||
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
||||
return oid.ID{}, err
|
||||
}
|
||||
var id oid.ID
|
||||
id.SetSHA256(sha256.Sum256(b))
|
||||
|
||||
attrs := make([]object.Attribute, 0)
|
||||
|
||||
if err := t.objectPutErrors[prm.Filepath]; err != nil {
|
||||
return oid.ID{}, err
|
||||
}
|
||||
|
||||
if prm.Filepath != "" {
|
||||
a := object.NewAttribute()
|
||||
a.SetKey(object.AttributeFilePath)
|
||||
a.SetValue(prm.Filepath)
|
||||
attrs = append(attrs, *a)
|
||||
}
|
||||
|
||||
if prm.ClientCut {
|
||||
a := object.NewAttribute()
|
||||
a.SetKey("s3-client-cut")
|
||||
a.SetValue("true")
|
||||
attrs = append(attrs, *a)
|
||||
}
|
||||
|
||||
for i := range prm.Attributes {
|
||||
a := object.NewAttribute()
|
||||
a.SetKey(prm.Attributes[i][0])
|
||||
a.SetValue(prm.Attributes[i][1])
|
||||
attrs = append(attrs, *a)
|
||||
}
|
||||
|
||||
var owner user.ID
|
||||
user.IDFromKey(&owner, t.key.PrivateKey.PublicKey)
|
||||
|
||||
obj := object.New()
|
||||
obj.SetContainerID(prm.Container)
|
||||
obj.SetID(id)
|
||||
obj.SetPayloadSize(prm.PayloadSize)
|
||||
obj.SetAttributes(attrs...)
|
||||
obj.SetCreationEpoch(t.currentEpoch)
|
||||
obj.SetOwnerID(owner)
|
||||
t.currentEpoch++
|
||||
|
||||
if len(prm.Locks) > 0 {
|
||||
lock := new(object.Lock)
|
||||
lock.WriteMembers(prm.Locks)
|
||||
objectv2.WriteLock(obj.ToV2(), (objectv2.Lock)(*lock))
|
||||
}
|
||||
|
||||
if prm.Payload != nil {
|
||||
all, err := io.ReadAll(prm.Payload)
|
||||
if err != nil {
|
||||
return oid.ID{}, err
|
||||
}
|
||||
obj.SetPayload(all)
|
||||
obj.SetPayloadSize(uint64(len(all)))
|
||||
var hash checksum.Checksum
|
||||
checksum.Calculate(&hash, checksum.SHA256, all)
|
||||
obj.SetPayloadChecksum(hash)
|
||||
}
|
||||
|
||||
cnrID, _ := obj.ContainerID()
|
||||
objID, _ := obj.ID()
|
||||
|
||||
addr := newAddress(cnrID, objID)
|
||||
t.objects[addr.EncodeToString()] = obj
|
||||
return objID, nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
|
||||
var addr oid.Address
|
||||
addr.SetContainer(prm.Container)
|
||||
addr.SetObject(prm.Object)
|
||||
|
||||
if err := t.objectErrors[addr.EncodeToString()]; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if obj, ok := t.objects[addr.EncodeToString()]; ok {
|
||||
owner := getBearerOwner(ctx)
|
||||
if !t.checkAccess(prm.Container, owner, eacl.OperationDelete, obj) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
delete(t.objects, addr.EncodeToString())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) TimeToEpoch(_ context.Context, now, futureTime time.Time) (uint64, uint64, error) {
|
||||
return t.currentEpoch, t.currentEpoch + uint64(futureTime.Sub(now).Seconds()), nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) AllObjects(cnrID cid.ID) []oid.ID {
|
||||
result := make([]oid.ID, 0, len(t.objects))
|
||||
|
||||
for _, val := range t.objects {
|
||||
objCnrID, _ := val.ContainerID()
|
||||
objObjID, _ := val.ID()
|
||||
if cnrID.Equals(objCnrID) {
|
||||
result = append(result, objObjID)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *session.Container) error {
|
||||
cnrID, ok := table.CID()
|
||||
if !ok {
|
||||
return errors.New("invalid cid")
|
||||
}
|
||||
|
||||
if _, ok = t.containers[cnrID.EncodeToString()]; !ok {
|
||||
return errors.New("not found")
|
||||
}
|
||||
|
||||
t.eaclTables[cnrID.EncodeToString()] = &table
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) ContainerEACL(_ context.Context, prm PrmContainerEACL) (*eacl.Table, error) {
|
||||
table, ok := t.eaclTables[prm.ContainerID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, errors.New("not found")
|
||||
}
|
||||
|
||||
return table, nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) ([]oid.ID, error) {
|
||||
filters := object.NewSearchFilters()
|
||||
filters.AddRootFilter()
|
||||
|
||||
if prm.ExactAttribute[0] != "" {
|
||||
filters.AddFilter(prm.ExactAttribute[0], prm.ExactAttribute[1], object.MatchStringEqual)
|
||||
}
|
||||
|
||||
cidStr := prm.Container.EncodeToString()
|
||||
|
||||
var res []oid.ID
|
||||
|
||||
if len(filters) == 1 {
|
||||
for k, v := range t.objects {
|
||||
if strings.Contains(k, cidStr) {
|
||||
id, _ := v.ID()
|
||||
res = append(res, id)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
filter := filters[1]
|
||||
if len(filters) != 2 || filter.Operation() != object.MatchStringEqual {
|
||||
return nil, fmt.Errorf("usupported filters")
|
||||
}
|
||||
|
||||
for k, v := range t.objects {
|
||||
if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) {
|
||||
id, _ := v.ID()
|
||||
res = append(res, id)
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) checkAccess(cnrID cid.ID, owner user.ID, op eacl.Operation, obj *object.Object) bool {
|
||||
cnr, ok := t.containers[cnrID.EncodeToString()]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if !cnr.BasicACL().Extendable() {
|
||||
return cnr.Owner().Equals(owner)
|
||||
}
|
||||
|
||||
table, ok := t.eaclTables[cnrID.EncodeToString()]
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, rec := range table.Records() {
|
||||
if rec.Operation() != op {
|
||||
continue
|
||||
}
|
||||
|
||||
if !matchTarget(rec, owner) {
|
||||
continue
|
||||
}
|
||||
|
||||
if matchFilter(rec.Filters(), obj) {
|
||||
return rec.Action() == eacl.ActionAllow
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func matchTarget(rec eacl.Record, owner user.ID) bool {
|
||||
for _, trgt := range rec.Targets() {
|
||||
if trgt.Role() == eacl.RoleOthers {
|
||||
return true
|
||||
}
|
||||
var targetOwner user.ID
|
||||
for _, pk := range eacl.TargetECDSAKeys(&trgt) {
|
||||
user.IDFromKey(&targetOwner, *pk)
|
||||
if targetOwner.Equals(owner) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func matchFilter(filters []eacl.Filter, obj *object.Object) bool {
|
||||
objID, _ := obj.ID()
|
||||
for _, f := range filters {
|
||||
fv2 := f.ToV2()
|
||||
if fv2.GetMatchType() != acl.MatchTypeStringEqual ||
|
||||
fv2.GetHeaderType() != acl.HeaderTypeObject ||
|
||||
fv2.GetKey() != acl.FilterObjectID ||
|
||||
fv2.GetValue() != objID.EncodeToString() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func getBearerOwner(ctx context.Context) user.ID {
|
||||
if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate.BearerToken != nil {
|
||||
return bearer.ResolveIssuer(*bd.Gate.BearerToken)
|
||||
}
|
||||
|
||||
return user.ID{}
|
||||
}
|
||||
|
||||
func isMatched(attributes []object.Attribute, filter object.SearchFilter) bool {
|
||||
for _, attr := range attributes {
|
||||
if attr.Key() == filter.Header() && attr.Value() == filter.Value() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
|
@ -4,8 +4,6 @@ import (
|
|||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
|
@ -13,20 +11,18 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"go.uber.org/zap"
|
||||
|
@ -48,33 +44,22 @@ type (
|
|||
Resolve(ctx context.Context, name string) (cid.ID, error)
|
||||
}
|
||||
|
||||
FeatureSettings interface {
|
||||
ClientCut() bool
|
||||
BufferMaxSizeForPut() uint64
|
||||
MD5Enabled() bool
|
||||
FormContainerZone(ns string) (zone string, isDefault bool)
|
||||
}
|
||||
|
||||
layer struct {
|
||||
frostFS FrostFS
|
||||
gateOwner user.ID
|
||||
log *zap.Logger
|
||||
anonKey AnonymousKey
|
||||
resolver BucketResolver
|
||||
ncontroller EventListener
|
||||
cache *Cache
|
||||
treeService TreeService
|
||||
features FeatureSettings
|
||||
}
|
||||
|
||||
Config struct {
|
||||
GateOwner user.ID
|
||||
ChainAddress string
|
||||
Cache *Cache
|
||||
Caches *CachesConfig
|
||||
AnonKey AnonymousKey
|
||||
Resolver BucketResolver
|
||||
TreeService TreeService
|
||||
Features FeatureSettings
|
||||
}
|
||||
|
||||
// AnonymousKey contains data for anonymous requests.
|
||||
|
@ -87,7 +72,7 @@ type (
|
|||
Range *RangeParams
|
||||
ObjectInfo *data.ObjectInfo
|
||||
BucketInfo *data.BucketInfo
|
||||
Versioned bool
|
||||
Writer io.Writer
|
||||
Encryption encryption.Params
|
||||
}
|
||||
|
||||
|
@ -98,6 +83,14 @@ type (
|
|||
VersionID string
|
||||
}
|
||||
|
||||
// ObjectVersion stores object version info.
|
||||
ObjectVersion struct {
|
||||
BktInfo *data.BucketInfo
|
||||
ObjectName string
|
||||
VersionID string
|
||||
NoErrorOnDeleteMarker bool
|
||||
}
|
||||
|
||||
// RangeParams stores range header request parameters.
|
||||
RangeParams struct {
|
||||
Start uint64
|
||||
|
@ -106,33 +99,20 @@ type (
|
|||
|
||||
// PutObjectParams stores object put request parameters.
|
||||
PutObjectParams struct {
|
||||
BktInfo *data.BucketInfo
|
||||
Object string
|
||||
Size uint64
|
||||
Reader io.Reader
|
||||
Header map[string]string
|
||||
Lock *data.ObjectLock
|
||||
Encryption encryption.Params
|
||||
CopiesNumbers []uint32
|
||||
CompleteMD5Hash string
|
||||
ContentMD5 string
|
||||
ContentSHA256Hash string
|
||||
}
|
||||
|
||||
PutCombinedObjectParams struct {
|
||||
BktInfo *data.BucketInfo
|
||||
Object string
|
||||
Size uint64
|
||||
Header map[string]string
|
||||
Lock *data.ObjectLock
|
||||
Encryption encryption.Params
|
||||
BktInfo *data.BucketInfo
|
||||
Object string
|
||||
Size int64
|
||||
Reader io.Reader
|
||||
Header map[string]string
|
||||
Lock *data.ObjectLock
|
||||
Encryption encryption.Params
|
||||
CopiesNumber uint32
|
||||
}
|
||||
|
||||
DeleteObjectParams struct {
|
||||
BktInfo *data.BucketInfo
|
||||
Objects []*VersionedObject
|
||||
Settings *data.BucketSettings
|
||||
IsMultiple bool
|
||||
BktInfo *data.BucketInfo
|
||||
Objects []*VersionedObject
|
||||
Settings *data.BucketSettings
|
||||
}
|
||||
|
||||
// PutSettingsParams stores object copy request parameters.
|
||||
|
@ -143,36 +123,33 @@ type (
|
|||
|
||||
// PutCORSParams stores PutCORS request parameters.
|
||||
PutCORSParams struct {
|
||||
BktInfo *data.BucketInfo
|
||||
Reader io.Reader
|
||||
CopiesNumbers []uint32
|
||||
NewDecoder func(io.Reader) *xml.Decoder
|
||||
BktInfo *data.BucketInfo
|
||||
Reader io.Reader
|
||||
CopiesNumber uint32
|
||||
}
|
||||
|
||||
// CopyObjectParams stores object copy request parameters.
|
||||
CopyObjectParams struct {
|
||||
SrcVersioned bool
|
||||
SrcObject *data.ObjectInfo
|
||||
ScrBktInfo *data.BucketInfo
|
||||
DstBktInfo *data.BucketInfo
|
||||
DstObject string
|
||||
DstSize uint64
|
||||
Header map[string]string
|
||||
Range *RangeParams
|
||||
Lock *data.ObjectLock
|
||||
SrcEncryption encryption.Params
|
||||
DstEncryption encryption.Params
|
||||
CopiesNumbers []uint32
|
||||
SrcObject *data.ObjectInfo
|
||||
ScrBktInfo *data.BucketInfo
|
||||
DstBktInfo *data.BucketInfo
|
||||
DstObject string
|
||||
SrcSize int64
|
||||
Header map[string]string
|
||||
Range *RangeParams
|
||||
Lock *data.ObjectLock
|
||||
Encryption encryption.Params
|
||||
CopiesNuber uint32
|
||||
}
|
||||
// CreateBucketParams stores bucket create request parameters.
|
||||
CreateBucketParams struct {
|
||||
Name string
|
||||
Namespace string
|
||||
Policy netmap.PlacementPolicy
|
||||
EACL *eacl.Table
|
||||
SessionContainerCreation *session.Container
|
||||
SessionEACL *session.Container
|
||||
LocationConstraint string
|
||||
ObjectLockEnabled bool
|
||||
APEEnabled bool
|
||||
}
|
||||
// PutBucketACLParams stores put bucket acl request parameters.
|
||||
PutBucketACLParams struct {
|
||||
|
@ -206,13 +183,6 @@ type (
|
|||
Error error
|
||||
}
|
||||
|
||||
ObjectPayload struct {
|
||||
r io.Reader
|
||||
params getParams
|
||||
encrypted bool
|
||||
decryptedLen uint64
|
||||
}
|
||||
|
||||
// Client provides S3 API client interface.
|
||||
Client interface {
|
||||
Initialize(ctx context.Context, c EventListener) error
|
||||
|
@ -227,26 +197,25 @@ type (
|
|||
|
||||
ListBuckets(ctx context.Context) ([]*data.BucketInfo, error)
|
||||
GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error)
|
||||
ResolveCID(ctx context.Context, name string) (cid.ID, error)
|
||||
GetBucketACL(ctx context.Context, bktInfo *data.BucketInfo) (*BucketACL, error)
|
||||
PutBucketACL(ctx context.Context, p *PutBucketACLParams) error
|
||||
CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error)
|
||||
DeleteBucket(ctx context.Context, p *DeleteBucketParams) error
|
||||
|
||||
GetObject(ctx context.Context, p *GetObjectParams) (*ObjectPayload, error)
|
||||
GetObject(ctx context.Context, p *GetObjectParams) error
|
||||
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error)
|
||||
GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error)
|
||||
|
||||
GetLockInfo(ctx context.Context, obj *data.ObjectVersion) (*data.LockInfo, error)
|
||||
GetLockInfo(ctx context.Context, obj *ObjectVersion) (*data.LockInfo, error)
|
||||
PutLockInfo(ctx context.Context, p *PutLockInfoParams) error
|
||||
|
||||
GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error)
|
||||
PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error
|
||||
DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error
|
||||
|
||||
GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error)
|
||||
PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingParams) (*data.NodeVersion, error)
|
||||
DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion) (*data.NodeVersion, error)
|
||||
GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) (string, map[string]string, error)
|
||||
PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (*data.NodeVersion, error)
|
||||
DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*data.NodeVersion, error)
|
||||
|
||||
PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error)
|
||||
|
||||
|
@ -272,7 +241,7 @@ type (
|
|||
// Compound methods for optimizations
|
||||
|
||||
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
|
||||
GetObjectTaggingAndLock(ctx context.Context, p *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error)
|
||||
GetObjectTaggingAndLock(ctx context.Context, p *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error)
|
||||
}
|
||||
)
|
||||
|
||||
|
@ -289,13 +258,6 @@ const (
|
|||
AttributeFrostfsCopiesNumber = "frostfs-copies-number" // such format to match X-Amz-Meta-Frostfs-Copies-Number header
|
||||
)
|
||||
|
||||
var EncryptionMetadata = map[string]struct{}{
|
||||
AttributeEncryptionAlgorithm: {},
|
||||
AttributeDecryptedSize: {},
|
||||
AttributeHMACSalt: {},
|
||||
AttributeHMACKey: {},
|
||||
}
|
||||
|
||||
func (t *VersionedObject) String() string {
|
||||
return t.Name + ":" + t.VersionID
|
||||
}
|
||||
|
@ -304,22 +266,16 @@ func (f MsgHandlerFunc) HandleMessage(ctx context.Context, msg *nats.Msg) error
|
|||
return f(ctx, msg)
|
||||
}
|
||||
|
||||
func (p HeadObjectParams) Versioned() bool {
|
||||
return len(p.VersionID) > 0
|
||||
}
|
||||
|
||||
// NewLayer creates an instance of a layer. It checks credentials
|
||||
// and establishes gRPC connection with the node.
|
||||
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
|
||||
return &layer{
|
||||
frostFS: frostFS,
|
||||
log: log,
|
||||
gateOwner: config.GateOwner,
|
||||
anonKey: config.AnonKey,
|
||||
resolver: config.Resolver,
|
||||
cache: config.Cache,
|
||||
cache: NewCache(config.Caches),
|
||||
treeService: config.TreeService,
|
||||
features: config.Features,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -346,22 +302,22 @@ func (n *layer) IsNotificationEnabled() bool {
|
|||
|
||||
// IsAuthenticatedRequest checks if access box exists in the current request.
|
||||
func IsAuthenticatedRequest(ctx context.Context) bool {
|
||||
_, err := middleware.GetBoxData(ctx)
|
||||
return err == nil
|
||||
_, ok := ctx.Value(api.BoxData).(*accessbox.Box)
|
||||
return ok
|
||||
}
|
||||
|
||||
// TimeNow returns client time from request or time.Now().
|
||||
func TimeNow(ctx context.Context) time.Time {
|
||||
if now, err := middleware.GetClientTime(ctx); err == nil {
|
||||
if now, ok := ctx.Value(api.ClientTime).(time.Time); ok {
|
||||
return now
|
||||
}
|
||||
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// BearerOwner returns owner id from BearerToken (context) or from client owner.
|
||||
func (n *layer) BearerOwner(ctx context.Context) user.ID {
|
||||
if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate.BearerToken != nil {
|
||||
// Owner returns owner id from BearerToken (context) or from client owner.
|
||||
func (n *layer) Owner(ctx context.Context) user.ID {
|
||||
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
||||
return bearer.ResolveIssuer(*bd.Gate.BearerToken)
|
||||
}
|
||||
|
||||
|
@ -371,26 +327,9 @@ func (n *layer) BearerOwner(ctx context.Context) user.ID {
|
|||
return ownerID
|
||||
}
|
||||
|
||||
// SessionTokenForRead returns session container token.
|
||||
func (n *layer) SessionTokenForRead(ctx context.Context) *session.Container {
|
||||
if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate != nil {
|
||||
return bd.Gate.SessionToken()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *layer) reqLogger(ctx context.Context) *zap.Logger {
|
||||
reqLogger := middleware.GetReqLog(ctx)
|
||||
if reqLogger != nil {
|
||||
return reqLogger
|
||||
}
|
||||
return n.log
|
||||
}
|
||||
|
||||
func (n *layer) prepareAuthParameters(ctx context.Context, prm *PrmAuth, bktOwner user.ID) {
|
||||
if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate.BearerToken != nil {
|
||||
if bd.Gate.BearerToken.Impersonate() || bktOwner.Equals(bearer.ResolveIssuer(*bd.Gate.BearerToken)) {
|
||||
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
||||
if bktOwner.Equals(bearer.ResolveIssuer(*bd.Gate.BearerToken)) {
|
||||
prm.BearerToken = bd.Gate.BearerToken
|
||||
return
|
||||
}
|
||||
|
@ -406,44 +345,17 @@ func (n *layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInf
|
|||
return nil, fmt.Errorf("unescape bucket name: %w", err)
|
||||
}
|
||||
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
zone, _ := n.features.FormContainerZone(reqInfo.Namespace)
|
||||
|
||||
if bktInfo := n.cache.GetBucket(zone, name); bktInfo != nil {
|
||||
if bktInfo := n.cache.GetBucket(name); bktInfo != nil {
|
||||
return bktInfo, nil
|
||||
}
|
||||
|
||||
containerID, err := n.ResolveBucket(ctx, name)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchBucket), err.Error())
|
||||
}
|
||||
return nil, err
|
||||
n.log.Debug("bucket not found", zap.Error(err))
|
||||
return nil, errors.GetAPIError(errors.ErrNoSuchBucket)
|
||||
}
|
||||
|
||||
prm := PrmContainer{
|
||||
ContainerID: containerID,
|
||||
SessionToken: n.SessionTokenForRead(ctx),
|
||||
}
|
||||
|
||||
return n.containerInfo(ctx, prm)
|
||||
}
|
||||
|
||||
// ResolveCID returns container id by name.
|
||||
func (n *layer) ResolveCID(ctx context.Context, name string) (cid.ID, error) {
|
||||
name, err := url.QueryUnescape(name)
|
||||
if err != nil {
|
||||
return cid.ID{}, fmt.Errorf("unescape bucket name: %w", err)
|
||||
}
|
||||
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
zone, _ := n.features.FormContainerZone(reqInfo.Namespace)
|
||||
|
||||
if bktInfo := n.cache.GetBucket(zone, name); bktInfo != nil {
|
||||
return bktInfo.CID, nil
|
||||
}
|
||||
|
||||
return n.ResolveBucket(ctx, name)
|
||||
return n.containerInfo(ctx, containerID)
|
||||
}
|
||||
|
||||
// GetBucketACL returns bucket acl info by name.
|
||||
|
@ -471,10 +383,10 @@ func (n *layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
|
|||
}
|
||||
|
||||
// GetObject from storage.
|
||||
func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) (*ObjectPayload, error) {
|
||||
func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
|
||||
var params getParams
|
||||
|
||||
params.objInfo = p.ObjectInfo
|
||||
params.oid = p.ObjectInfo.ID
|
||||
params.bktInfo = p.BucketInfo
|
||||
|
||||
var decReader *encryption.Decrypter
|
||||
|
@ -482,7 +394,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) (*ObjectPaylo
|
|||
var err error
|
||||
decReader, err = getDecrypter(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating decrypter: %w", err)
|
||||
return fmt.Errorf("creating decrypter: %w", err)
|
||||
}
|
||||
params.off = decReader.EncryptedOffset()
|
||||
params.ln = decReader.EncryptedLength()
|
||||
|
@ -496,58 +408,32 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) (*ObjectPaylo
|
|||
}
|
||||
}
|
||||
|
||||
r, err := n.initObjectPayloadReader(ctx, params)
|
||||
payload, err := n.initObjectPayloadReader(ctx, params)
|
||||
if err != nil {
|
||||
if client.IsErrObjectNotFound(err) {
|
||||
if p.Versioned {
|
||||
err = fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchVersion), err.Error())
|
||||
} else {
|
||||
err = fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchKey), err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("init object payload reader: %w", err)
|
||||
return fmt.Errorf("init object payload reader: %w", err)
|
||||
}
|
||||
|
||||
var decryptedLen uint64
|
||||
if decReader != nil {
|
||||
if err = decReader.SetReader(r); err != nil {
|
||||
return nil, fmt.Errorf("set reader to decrypter: %w", err)
|
||||
}
|
||||
r = io.LimitReader(decReader, int64(decReader.DecryptedLength()))
|
||||
decryptedLen = decReader.DecryptedLength()
|
||||
}
|
||||
|
||||
return &ObjectPayload{
|
||||
r: r,
|
||||
params: params,
|
||||
encrypted: decReader != nil,
|
||||
decryptedLen: decryptedLen,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Read implements io.Reader. If you want to use ObjectPayload as io.Reader
|
||||
// you must not use ObjectPayload.StreamTo method and vice versa.
|
||||
func (o *ObjectPayload) Read(p []byte) (int, error) {
|
||||
return o.r.Read(p)
|
||||
}
|
||||
|
||||
// StreamTo reads all payload to provided writer.
|
||||
// If you want to use this method you must not use ObjectPayload.Read and vice versa.
|
||||
func (o *ObjectPayload) StreamTo(w io.Writer) error {
|
||||
bufSize := uint64(32 * 1024) // configure?
|
||||
if o.params.ln != 0 && o.params.ln < bufSize {
|
||||
bufSize = o.params.ln
|
||||
if params.ln != 0 && params.ln < bufSize {
|
||||
bufSize = params.ln
|
||||
}
|
||||
|
||||
// alloc buffer for copying
|
||||
buf := make([]byte, bufSize) // sync-pool it?
|
||||
|
||||
r := payload
|
||||
if decReader != nil {
|
||||
if err = decReader.SetReader(payload); err != nil {
|
||||
return fmt.Errorf("set reader to decrypter: %w", err)
|
||||
}
|
||||
r = io.LimitReader(decReader, int64(decReader.DecryptedLength()))
|
||||
}
|
||||
|
||||
// copy full payload
|
||||
written, err := io.CopyBuffer(w, o.r, buf)
|
||||
written, err := io.CopyBuffer(p.Writer, r, buf)
|
||||
if err != nil {
|
||||
if o.encrypted {
|
||||
return fmt.Errorf("copy object payload written: '%d', decLength: '%d', params.ln: '%d' : %w", written, o.decryptedLen, o.params.ln, err)
|
||||
if decReader != nil {
|
||||
return fmt.Errorf("copy object payload written: '%d', decLength: '%d', params.ln: '%d' : %w", written, decReader.DecryptedLength(), params.ln, err)
|
||||
}
|
||||
return fmt.Errorf("copy object payload written: '%d': %w", written, err)
|
||||
}
|
||||
|
@ -599,17 +485,21 @@ func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams)
|
|||
var objInfo *data.ExtendedObjectInfo
|
||||
var err error
|
||||
|
||||
if p.Versioned() {
|
||||
objInfo, err = n.headVersion(ctx, p.BktInfo, p)
|
||||
} else {
|
||||
if len(p.VersionID) == 0 {
|
||||
objInfo, err = n.headLastVersionIfNotDeleted(ctx, p.BktInfo, p.Object)
|
||||
} else {
|
||||
objInfo, err = n.headVersion(ctx, p.BktInfo, p)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Debug(logs.GetObject,
|
||||
reqInfo := api.GetReqInfo(ctx)
|
||||
n.log.Debug("get object",
|
||||
zap.String("reqId", reqInfo.RequestID),
|
||||
zap.String("bucket", p.BktInfo.Name),
|
||||
zap.Stringer("cid", p.BktInfo.CID),
|
||||
zap.String("object", objInfo.ObjectInfo.Name),
|
||||
zap.Stringer("oid", objInfo.ObjectInfo.ID))
|
||||
|
||||
return objInfo, nil
|
||||
|
@ -617,25 +507,30 @@ func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams)
|
|||
|
||||
// CopyObject from one bucket into another bucket.
|
||||
func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||
objPayload, err := n.GetObject(ctx, &GetObjectParams{
|
||||
ObjectInfo: p.SrcObject,
|
||||
Versioned: p.SrcVersioned,
|
||||
Range: p.Range,
|
||||
BucketInfo: p.ScrBktInfo,
|
||||
Encryption: p.SrcEncryption,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get object to copy: %w", err)
|
||||
}
|
||||
pr, pw := io.Pipe()
|
||||
|
||||
go func() {
|
||||
err := n.GetObject(ctx, &GetObjectParams{
|
||||
ObjectInfo: p.SrcObject,
|
||||
Writer: pw,
|
||||
Range: p.Range,
|
||||
BucketInfo: p.ScrBktInfo,
|
||||
Encryption: p.Encryption,
|
||||
})
|
||||
|
||||
if err = pw.CloseWithError(err); err != nil {
|
||||
n.log.Error("could not get object", zap.Error(err))
|
||||
}
|
||||
}()
|
||||
|
||||
return n.PutObject(ctx, &PutObjectParams{
|
||||
BktInfo: p.DstBktInfo,
|
||||
Object: p.DstObject,
|
||||
Size: p.DstSize,
|
||||
Reader: objPayload,
|
||||
Header: p.Header,
|
||||
Encryption: p.DstEncryption,
|
||||
CopiesNumbers: p.CopiesNumbers,
|
||||
BktInfo: p.DstBktInfo,
|
||||
Object: p.DstObject,
|
||||
Size: p.SrcSize,
|
||||
Reader: pr,
|
||||
Header: p.Header,
|
||||
Encryption: p.Encryption,
|
||||
CopiesNumber: p.CopiesNuber,
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -654,11 +549,11 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
|||
if len(obj.VersionID) != 0 || settings.Unversioned() {
|
||||
var nodeVersion *data.NodeVersion
|
||||
if nodeVersion, obj.Error = n.getNodeVersionToDelete(ctx, bkt, obj); obj.Error != nil {
|
||||
return n.handleNotFoundError(bkt, obj)
|
||||
return dismissNotFoundError(obj)
|
||||
}
|
||||
|
||||
if obj.DeleteMarkVersion, obj.Error = n.removeOldVersion(ctx, bkt, nodeVersion, obj); obj.Error != nil {
|
||||
return n.handleObjectDeleteErrors(ctx, bkt, obj, nodeVersion.ID)
|
||||
return obj
|
||||
}
|
||||
|
||||
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID)
|
||||
|
@ -666,38 +561,21 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
|||
return obj
|
||||
}
|
||||
|
||||
lastVersion, err := n.getLastNodeVersion(ctx, bkt, obj)
|
||||
if err != nil {
|
||||
obj.Error = err
|
||||
return n.handleNotFoundError(bkt, obj)
|
||||
}
|
||||
var newVersion *data.NodeVersion
|
||||
|
||||
if settings.VersioningSuspended() {
|
||||
obj.VersionID = data.UnversionedObjectVersionID
|
||||
|
||||
var nullVersionToDelete *data.NodeVersion
|
||||
if lastVersion.IsUnversioned {
|
||||
if !lastVersion.IsDeleteMarker {
|
||||
nullVersionToDelete = lastVersion
|
||||
}
|
||||
} else if nullVersionToDelete, obj.Error = n.getNodeVersionToDelete(ctx, bkt, obj); obj.Error != nil {
|
||||
if !isNotFoundError(obj.Error) {
|
||||
return obj
|
||||
}
|
||||
var nodeVersion *data.NodeVersion
|
||||
if nodeVersion, obj.Error = n.getNodeVersionToDelete(ctx, bkt, obj); obj.Error != nil {
|
||||
return dismissNotFoundError(obj)
|
||||
}
|
||||
|
||||
if nullVersionToDelete != nil {
|
||||
if obj.DeleteMarkVersion, obj.Error = n.removeOldVersion(ctx, bkt, nullVersionToDelete, obj); obj.Error != nil {
|
||||
return n.handleObjectDeleteErrors(ctx, bkt, obj, nullVersionToDelete.ID)
|
||||
}
|
||||
if obj.DeleteMarkVersion, obj.Error = n.removeOldVersion(ctx, bkt, nodeVersion, obj); obj.Error != nil {
|
||||
return obj
|
||||
}
|
||||
}
|
||||
|
||||
if lastVersion.IsDeleteMarker {
|
||||
obj.DeleteMarkVersion = lastVersion.OID.EncodeToString()
|
||||
return obj
|
||||
}
|
||||
|
||||
randOID, err := getRandomOID()
|
||||
if err != nil {
|
||||
obj.Error = fmt.Errorf("couldn't get random oid: %w", err)
|
||||
|
@ -705,14 +583,15 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
|||
}
|
||||
|
||||
obj.DeleteMarkVersion = randOID.EncodeToString()
|
||||
now := TimeNow(ctx)
|
||||
newVersion := &data.NodeVersion{
|
||||
|
||||
newVersion = &data.NodeVersion{
|
||||
BaseNodeVersion: data.BaseNodeVersion{
|
||||
OID: randOID,
|
||||
FilePath: obj.Name,
|
||||
Created: &now,
|
||||
Owner: &n.gateOwner,
|
||||
IsDeleteMarker: true,
|
||||
OID: randOID,
|
||||
FilePath: obj.Name,
|
||||
},
|
||||
DeleteMarker: &data.DeleteMarkerInfo{
|
||||
Created: TimeNow(ctx),
|
||||
Owner: n.Owner(ctx),
|
||||
},
|
||||
IsUnversioned: settings.VersioningSuspended(),
|
||||
}
|
||||
|
@ -726,39 +605,17 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
|||
return obj
|
||||
}
|
||||
|
||||
func (n *layer) handleNotFoundError(bkt *data.BucketInfo, obj *VersionedObject) *VersionedObject {
|
||||
if isNotFoundError(obj.Error) {
|
||||
func dismissNotFoundError(obj *VersionedObject) *VersionedObject {
|
||||
if errors.IsS3Error(obj.Error, errors.ErrNoSuchKey) ||
|
||||
errors.IsS3Error(obj.Error, errors.ErrNoSuchVersion) {
|
||||
obj.Error = nil
|
||||
n.cache.CleanListCacheEntriesContainingObject(obj.Name, bkt.CID)
|
||||
n.cache.DeleteObjectName(bkt.CID, bkt.Name, obj.Name)
|
||||
}
|
||||
|
||||
return obj
|
||||
}
|
||||
|
||||
func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject, nodeID uint64) *VersionedObject {
|
||||
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
|
||||
return obj
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
|
||||
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
|
||||
|
||||
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeID)
|
||||
if obj.Error == nil {
|
||||
n.cache.DeleteObjectName(bkt.CID, bkt.Name, obj.Name)
|
||||
}
|
||||
|
||||
return obj
|
||||
}
|
||||
|
||||
func isNotFoundError(err error) bool {
|
||||
return errors.IsS3Error(err, errors.ErrNoSuchKey) ||
|
||||
errors.IsS3Error(err, errors.ErrNoSuchVersion)
|
||||
}
|
||||
|
||||
func (n *layer) getNodeVersionToDelete(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.NodeVersion, error) {
|
||||
objVersion := &data.ObjectVersion{
|
||||
objVersion := &ObjectVersion{
|
||||
BktInfo: bkt,
|
||||
ObjectName: obj.Name,
|
||||
VersionID: obj.VersionID,
|
||||
|
@ -768,63 +625,18 @@ func (n *layer) getNodeVersionToDelete(ctx context.Context, bkt *data.BucketInfo
|
|||
return n.getNodeVersion(ctx, objVersion)
|
||||
}
|
||||
|
||||
func (n *layer) getLastNodeVersion(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.NodeVersion, error) {
|
||||
objVersion := &data.ObjectVersion{
|
||||
BktInfo: bkt,
|
||||
ObjectName: obj.Name,
|
||||
VersionID: "",
|
||||
NoErrorOnDeleteMarker: true,
|
||||
}
|
||||
|
||||
return n.getNodeVersion(ctx, objVersion)
|
||||
}
|
||||
|
||||
func (n *layer) removeOldVersion(ctx context.Context, bkt *data.BucketInfo, nodeVersion *data.NodeVersion, obj *VersionedObject) (string, error) {
|
||||
if nodeVersion.IsDeleteMarker {
|
||||
if nodeVersion.IsDeleteMarker() {
|
||||
return obj.VersionID, nil
|
||||
}
|
||||
|
||||
if nodeVersion.IsCombined {
|
||||
return "", n.removeCombinedObject(ctx, bkt, nodeVersion)
|
||||
}
|
||||
|
||||
return "", n.objectDelete(ctx, bkt, nodeVersion.OID)
|
||||
}
|
||||
|
||||
func (n *layer) removeCombinedObject(ctx context.Context, bkt *data.BucketInfo, nodeVersion *data.NodeVersion) error {
|
||||
combinedObj, err := n.objectGet(ctx, bkt, nodeVersion.OID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get combined object '%s': %w", nodeVersion.OID.EncodeToString(), err)
|
||||
}
|
||||
|
||||
var parts []*data.PartInfo
|
||||
if err = json.Unmarshal(combinedObj.Payload(), &parts); err != nil {
|
||||
return fmt.Errorf("unmarshal combined object parts: %w", err)
|
||||
}
|
||||
|
||||
for _, part := range parts {
|
||||
if err = n.objectDelete(ctx, bkt, part.OID); err == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if !client.IsErrObjectAlreadyRemoved(err) && !client.IsErrObjectNotFound(err) {
|
||||
return fmt.Errorf("couldn't delete part '%s': %w", part.OID.EncodeToString(), err)
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Warn(logs.CouldntDeletePart, zap.String("cid", bkt.CID.EncodeToString()),
|
||||
zap.String("oid", part.OID.EncodeToString()), zap.Int("part number", part.Number), zap.Error(err))
|
||||
}
|
||||
|
||||
return n.objectDelete(ctx, bkt, nodeVersion.OID)
|
||||
}
|
||||
|
||||
// DeleteObjects from the storage.
|
||||
func (n *layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*VersionedObject {
|
||||
for i, obj := range p.Objects {
|
||||
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj)
|
||||
if p.IsMultiple && p.Objects[i].Error != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error))
|
||||
}
|
||||
}
|
||||
|
||||
return p.Objects
|
||||
|
@ -853,25 +665,22 @@ func (n *layer) ResolveBucket(ctx context.Context, name string) (cid.ID, error)
|
|||
return cid.ID{}, err
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID))
|
||||
reqInfo := api.GetReqInfo(ctx)
|
||||
n.log.Info("resolve bucket", zap.String("reqId", reqInfo.RequestID), zap.String("bucket", name), zap.Stringer("cid", cnrID))
|
||||
}
|
||||
|
||||
return cnrID, nil
|
||||
}
|
||||
|
||||
func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||
res, _, err := n.getAllObjectsVersions(ctx, commonVersionsListingParams{
|
||||
BktInfo: p.BktInfo,
|
||||
MaxKeys: 1,
|
||||
})
|
||||
|
||||
nodeVersions, err := n.bucketNodeVersions(ctx, p.BktInfo, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(res) != 0 {
|
||||
if len(nodeVersions) != 0 {
|
||||
return errors.GetAPIError(errors.ErrBucketNotEmpty)
|
||||
}
|
||||
|
||||
n.cache.DeleteBucket(p.BktInfo)
|
||||
n.cache.DeleteBucket(p.BktInfo.Name)
|
||||
return n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
||||
}
|
||||
|
|
|
@ -1,725 +0,0 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type (
|
||||
// ListObjectsParamsCommon contains common parameters for ListObjectsV1 and ListObjectsV2.
|
||||
ListObjectsParamsCommon struct {
|
||||
BktInfo *data.BucketInfo
|
||||
Delimiter string
|
||||
Encode string
|
||||
MaxKeys int
|
||||
Prefix string
|
||||
}
|
||||
|
||||
// ListObjectsParamsV1 contains params for ListObjectsV1.
|
||||
ListObjectsParamsV1 struct {
|
||||
ListObjectsParamsCommon
|
||||
Marker string
|
||||
}
|
||||
|
||||
// ListObjectsParamsV2 contains params for ListObjectsV2.
|
||||
ListObjectsParamsV2 struct {
|
||||
ListObjectsParamsCommon
|
||||
ContinuationToken string
|
||||
StartAfter string
|
||||
FetchOwner bool
|
||||
}
|
||||
|
||||
// ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2.
|
||||
ListObjectsInfo struct {
|
||||
Prefixes []string
|
||||
Objects []*data.ExtendedNodeVersion
|
||||
IsTruncated bool
|
||||
}
|
||||
|
||||
// ListObjectsInfoV1 holds data which ListObjectsV1 returns.
|
||||
ListObjectsInfoV1 struct {
|
||||
ListObjectsInfo
|
||||
NextMarker string
|
||||
}
|
||||
|
||||
// ListObjectsInfoV2 holds data which ListObjectsV2 returns.
|
||||
ListObjectsInfoV2 struct {
|
||||
ListObjectsInfo
|
||||
NextContinuationToken string
|
||||
}
|
||||
|
||||
// ListObjectVersionsInfo stores info and list of objects versions.
|
||||
ListObjectVersionsInfo struct {
|
||||
CommonPrefixes []string
|
||||
IsTruncated bool
|
||||
KeyMarker string
|
||||
NextKeyMarker string
|
||||
NextVersionIDMarker string
|
||||
Version []*data.ExtendedNodeVersion
|
||||
DeleteMarker []*data.ExtendedNodeVersion
|
||||
VersionIDMarker string
|
||||
}
|
||||
|
||||
commonVersionsListingParams struct {
|
||||
BktInfo *data.BucketInfo
|
||||
Delimiter string
|
||||
Prefix string
|
||||
MaxKeys int
|
||||
Marker string
|
||||
Bookmark string
|
||||
}
|
||||
|
||||
commonLatestVersionsListingParams struct {
|
||||
commonVersionsListingParams
|
||||
ListType ListType
|
||||
}
|
||||
)
|
||||
|
||||
type ListType int
|
||||
|
||||
const (
|
||||
ListObjectsV1Type ListType = iota + 1
|
||||
ListObjectsV2Type ListType = iota + 1
|
||||
)
|
||||
|
||||
// ListObjectsV1 returns objects in a bucket for requests of Version 1.
|
||||
func (n *layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error) {
|
||||
var result ListObjectsInfoV1
|
||||
|
||||
prm := commonLatestVersionsListingParams{
|
||||
commonVersionsListingParams: commonVersionsListingParams{
|
||||
BktInfo: p.BktInfo,
|
||||
Delimiter: p.Delimiter,
|
||||
Prefix: p.Prefix,
|
||||
MaxKeys: p.MaxKeys,
|
||||
Marker: p.Marker,
|
||||
Bookmark: p.Marker,
|
||||
},
|
||||
ListType: ListObjectsV1Type,
|
||||
}
|
||||
|
||||
objects, next, err := n.getLatestObjectsVersions(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if next != nil {
|
||||
result.IsTruncated = true
|
||||
result.NextMarker = objects[len(objects)-1].Name()
|
||||
}
|
||||
|
||||
result.Prefixes, result.Objects = triageExtendedObjects(objects)
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// ListObjectsV2 returns objects in a bucket for requests of Version 2.
|
||||
func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) {
|
||||
var result ListObjectsInfoV2
|
||||
|
||||
prm := commonLatestVersionsListingParams{
|
||||
commonVersionsListingParams: commonVersionsListingParams{
|
||||
BktInfo: p.BktInfo,
|
||||
Delimiter: p.Delimiter,
|
||||
Prefix: p.Prefix,
|
||||
MaxKeys: p.MaxKeys,
|
||||
Marker: p.StartAfter,
|
||||
Bookmark: p.ContinuationToken,
|
||||
},
|
||||
ListType: ListObjectsV2Type,
|
||||
}
|
||||
|
||||
objects, next, err := n.getLatestObjectsVersions(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if next != nil {
|
||||
result.IsTruncated = true
|
||||
result.NextContinuationToken = next.NodeVersion.OID.EncodeToString()
|
||||
}
|
||||
|
||||
result.Prefixes, result.Objects = triageExtendedObjects(objects)
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
||||
prm := commonVersionsListingParams{
|
||||
BktInfo: p.BktInfo,
|
||||
Delimiter: p.Delimiter,
|
||||
Prefix: p.Prefix,
|
||||
MaxKeys: p.MaxKeys,
|
||||
Marker: p.KeyMarker,
|
||||
Bookmark: p.VersionIDMarker,
|
||||
}
|
||||
|
||||
objects, isTruncated, err := n.getAllObjectsVersions(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := &ListObjectVersionsInfo{
|
||||
KeyMarker: p.KeyMarker,
|
||||
VersionIDMarker: p.VersionIDMarker,
|
||||
IsTruncated: isTruncated,
|
||||
}
|
||||
|
||||
if res.IsTruncated {
|
||||
res.NextKeyMarker = objects[p.MaxKeys-1].NodeVersion.FilePath
|
||||
res.NextVersionIDMarker = objects[p.MaxKeys-1].NodeVersion.OID.EncodeToString()
|
||||
}
|
||||
|
||||
res.CommonPrefixes, objects = triageExtendedObjects(objects)
|
||||
res.Version, res.DeleteMarker = triageVersions(objects)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (n *layer) getLatestObjectsVersions(ctx context.Context, p commonLatestVersionsListingParams) (objects []*data.ExtendedNodeVersion, next *data.ExtendedNodeVersion, err error) {
|
||||
if p.MaxKeys == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
session, err := n.getListLatestVersionsSession(ctx, p)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
generator, errorCh := nodesGeneratorStream(ctx, p.commonVersionsListingParams, session)
|
||||
objOutCh, err := n.initWorkerPool(ctx, 2, p.commonVersionsListingParams, generator)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to init worker pool: %w", err)
|
||||
}
|
||||
|
||||
objects = make([]*data.ExtendedNodeVersion, 0, p.MaxKeys+1)
|
||||
objects = append(objects, session.Next...)
|
||||
for obj := range objOutCh {
|
||||
objects = append(objects, obj)
|
||||
}
|
||||
|
||||
if err = <-errorCh; err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to get next object from stream: %w", err)
|
||||
}
|
||||
|
||||
sort.Slice(objects, func(i, j int) bool { return objects[i].NodeVersion.FilePath < objects[j].NodeVersion.FilePath })
|
||||
|
||||
if len(objects) > p.MaxKeys {
|
||||
next = objects[p.MaxKeys]
|
||||
n.putListLatestVersionsSession(ctx, p, session, objects)
|
||||
objects = objects[:p.MaxKeys]
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (n *layer) getAllObjectsVersions(ctx context.Context, p commonVersionsListingParams) ([]*data.ExtendedNodeVersion, bool, error) {
|
||||
if p.MaxKeys == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
session, err := n.getListAllVersionsSession(ctx, p)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
generator, errorCh := nodesGeneratorVersions(ctx, p, session)
|
||||
objOutCh, err := n.initWorkerPool(ctx, 2, p, generator)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
allObjects := handleGeneratedVersions(objOutCh, p, session)
|
||||
|
||||
sort.SliceStable(allObjects, func(i, j int) bool { return allObjects[i].NodeVersion.FilePath < allObjects[j].NodeVersion.FilePath })
|
||||
|
||||
if err = <-errorCh; err != nil {
|
||||
return nil, false, fmt.Errorf("failed to get next object from stream: %w", err)
|
||||
}
|
||||
|
||||
var isTruncated bool
|
||||
if len(allObjects) > p.MaxKeys {
|
||||
isTruncated = true
|
||||
n.putListAllVersionsSession(ctx, p, session, allObjects)
|
||||
allObjects = allObjects[:p.MaxKeys]
|
||||
}
|
||||
|
||||
return allObjects, isTruncated, nil
|
||||
}
|
||||
|
||||
func handleGeneratedVersions(objOutCh <-chan *data.ExtendedNodeVersion, p commonVersionsListingParams, session *data.ListSession) []*data.ExtendedNodeVersion {
|
||||
var lastName string
|
||||
var listRowStartIndex int
|
||||
allObjects := make([]*data.ExtendedNodeVersion, 0, p.MaxKeys)
|
||||
for eoi := range objOutCh {
|
||||
name := eoi.NodeVersion.FilePath
|
||||
if eoi.DirName != "" {
|
||||
name = eoi.DirName
|
||||
}
|
||||
|
||||
if lastName != name {
|
||||
formVersionsListRow(allObjects, listRowStartIndex, session)
|
||||
listRowStartIndex = len(allObjects)
|
||||
allObjects = append(allObjects, eoi)
|
||||
} else if eoi.DirName == "" {
|
||||
allObjects = append(allObjects, eoi)
|
||||
}
|
||||
lastName = name
|
||||
}
|
||||
|
||||
formVersionsListRow(allObjects, listRowStartIndex, session)
|
||||
|
||||
return allObjects
|
||||
}
|
||||
|
||||
func formVersionsListRow(objects []*data.ExtendedNodeVersion, rowStartIndex int, session *data.ListSession) {
|
||||
if len(objects) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
prevVersions := objects[rowStartIndex:]
|
||||
sort.Slice(prevVersions, func(i, j int) bool {
|
||||
return prevVersions[j].NodeVersion.Timestamp < prevVersions[i].NodeVersion.Timestamp // sort in reverse order to have last added first
|
||||
})
|
||||
|
||||
prevVersions[0].IsLatest = len(session.Next) == 0 || session.Next[0].NodeVersion.FilePath != prevVersions[0].NodeVersion.FilePath
|
||||
|
||||
for _, version := range prevVersions[1:] {
|
||||
version.IsLatest = false
|
||||
}
|
||||
}
|
||||
|
||||
func (n *layer) getListLatestVersionsSession(ctx context.Context, p commonLatestVersionsListingParams) (*data.ListSession, error) {
|
||||
return n.getListVersionsSession(ctx, p.commonVersionsListingParams, true)
|
||||
}
|
||||
|
||||
func (n *layer) getListAllVersionsSession(ctx context.Context, p commonVersionsListingParams) (*data.ListSession, error) {
|
||||
return n.getListVersionsSession(ctx, p, false)
|
||||
}
|
||||
|
||||
func (n *layer) getListVersionsSession(ctx context.Context, p commonVersionsListingParams, latestOnly bool) (*data.ListSession, error) {
|
||||
owner := n.BearerOwner(ctx)
|
||||
|
||||
cacheKey := cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, p.Bookmark)
|
||||
session := n.cache.GetListSession(owner, cacheKey)
|
||||
if session == nil {
|
||||
return n.initNewVersionsByPrefixSession(ctx, p, latestOnly)
|
||||
}
|
||||
|
||||
if session.Acquired.Swap(true) {
|
||||
return n.initNewVersionsByPrefixSession(ctx, p, latestOnly)
|
||||
}
|
||||
|
||||
// after reading next object from stream in session
|
||||
// the current cache value already doesn't match with next token in cache key
|
||||
n.cache.DeleteListSession(owner, cacheKey)
|
||||
|
||||
return session, nil
|
||||
}
|
||||
|
||||
func (n *layer) initNewVersionsByPrefixSession(ctx context.Context, p commonVersionsListingParams, latestOnly bool) (session *data.ListSession, err error) {
|
||||
session = &data.ListSession{NamesMap: make(map[string]struct{})}
|
||||
session.Context, session.Cancel = context.WithCancel(context.Background())
|
||||
|
||||
if bd, err := middleware.GetBoxData(ctx); err == nil {
|
||||
session.Context = middleware.SetBox(session.Context, &middleware.Box{AccessBox: bd})
|
||||
}
|
||||
|
||||
session.Stream, err = n.treeService.InitVersionsByPrefixStream(session.Context, p.BktInfo, p.Prefix, latestOnly)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return session, nil
|
||||
}
|
||||
|
||||
func (n *layer) putListLatestVersionsSession(ctx context.Context, p commonLatestVersionsListingParams, session *data.ListSession, allObjects []*data.ExtendedNodeVersion) {
|
||||
if len(allObjects) <= p.MaxKeys {
|
||||
return
|
||||
}
|
||||
|
||||
var cacheKey cache.ListSessionKey
|
||||
switch p.ListType {
|
||||
case ListObjectsV1Type:
|
||||
cacheKey = cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, allObjects[p.MaxKeys-1].Name())
|
||||
case ListObjectsV2Type:
|
||||
cacheKey = cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, allObjects[p.MaxKeys].NodeVersion.OID.EncodeToString())
|
||||
default:
|
||||
// should never happen
|
||||
panic("invalid list type")
|
||||
}
|
||||
|
||||
session.Acquired.Store(false)
|
||||
session.Next = []*data.ExtendedNodeVersion{allObjects[p.MaxKeys]}
|
||||
n.cache.PutListSession(n.BearerOwner(ctx), cacheKey, session)
|
||||
}
|
||||
|
||||
func (n *layer) putListAllVersionsSession(ctx context.Context, p commonVersionsListingParams, session *data.ListSession, allObjects []*data.ExtendedNodeVersion) {
|
||||
if len(allObjects) <= p.MaxKeys {
|
||||
return
|
||||
}
|
||||
|
||||
session.Acquired.Store(false)
|
||||
|
||||
session.Next = make([]*data.ExtendedNodeVersion, len(allObjects)-p.MaxKeys+1)
|
||||
session.Next[0] = allObjects[p.MaxKeys-1]
|
||||
for i, node := range allObjects[p.MaxKeys:] {
|
||||
session.Next[i+1] = node
|
||||
}
|
||||
|
||||
cacheKey := cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, session.Next[0].NodeVersion.OID.EncodeToString())
|
||||
n.cache.PutListSession(n.BearerOwner(ctx), cacheKey, session)
|
||||
}
|
||||
|
||||
func nodesGeneratorStream(ctx context.Context, p commonVersionsListingParams, stream *data.ListSession) (<-chan *data.ExtendedNodeVersion, <-chan error) {
|
||||
nodeCh := make(chan *data.ExtendedNodeVersion, 1000)
|
||||
errCh := make(chan error, 1)
|
||||
existed := stream.NamesMap
|
||||
|
||||
if len(stream.Next) != 0 {
|
||||
existed[continuationToken] = struct{}{}
|
||||
}
|
||||
|
||||
limit := p.MaxKeys
|
||||
if len(stream.Next) == 0 {
|
||||
limit++
|
||||
}
|
||||
|
||||
go func() {
|
||||
var generated int
|
||||
var err error
|
||||
|
||||
LOOP:
|
||||
for err == nil {
|
||||
node, err := stream.Stream.Next(ctx)
|
||||
if err != nil {
|
||||
if !errors.Is(err, io.EOF) {
|
||||
errCh <- fmt.Errorf("stream next: %w", err)
|
||||
}
|
||||
break LOOP
|
||||
}
|
||||
|
||||
nodeExt := &data.ExtendedNodeVersion{
|
||||
NodeVersion: node,
|
||||
IsLatest: true,
|
||||
DirName: tryDirectoryName(node, p.Prefix, p.Delimiter),
|
||||
}
|
||||
|
||||
if shouldSkip(nodeExt, p, existed) {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break LOOP
|
||||
case nodeCh <- nodeExt:
|
||||
generated++
|
||||
|
||||
if generated == limit { // we use maxKeys+1 to be able to know nextMarker/nextContinuationToken
|
||||
break LOOP
|
||||
}
|
||||
}
|
||||
}
|
||||
close(nodeCh)
|
||||
close(errCh)
|
||||
}()
|
||||
|
||||
return nodeCh, errCh
|
||||
}
|
||||
|
||||
func nodesGeneratorVersions(ctx context.Context, p commonVersionsListingParams, stream *data.ListSession) (<-chan *data.ExtendedNodeVersion, <-chan error) {
|
||||
nodeCh := make(chan *data.ExtendedNodeVersion, 1000)
|
||||
errCh := make(chan error, 1)
|
||||
existed := stream.NamesMap
|
||||
|
||||
delete(existed, continuationToken)
|
||||
|
||||
go func() {
|
||||
var (
|
||||
generated int
|
||||
ind int
|
||||
err error
|
||||
lastName string
|
||||
node *data.NodeVersion
|
||||
nodeExt *data.ExtendedNodeVersion
|
||||
)
|
||||
|
||||
LOOP:
|
||||
for err == nil {
|
||||
if ind < len(stream.Next) {
|
||||
nodeExt = stream.Next[ind]
|
||||
ind++
|
||||
} else {
|
||||
node, err = stream.Stream.Next(ctx)
|
||||
if err != nil {
|
||||
if !errors.Is(err, io.EOF) {
|
||||
errCh <- fmt.Errorf("stream next: %w", err)
|
||||
}
|
||||
break LOOP
|
||||
}
|
||||
|
||||
nodeExt = &data.ExtendedNodeVersion{
|
||||
NodeVersion: node,
|
||||
DirName: tryDirectoryName(node, p.Prefix, p.Delimiter),
|
||||
}
|
||||
}
|
||||
|
||||
if shouldSkipVersions(nodeExt, p, existed) {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break LOOP
|
||||
case nodeCh <- nodeExt:
|
||||
generated++
|
||||
if generated > p.MaxKeys && nodeExt.NodeVersion.FilePath != lastName {
|
||||
break LOOP
|
||||
}
|
||||
lastName = nodeExt.NodeVersion.FilePath
|
||||
}
|
||||
}
|
||||
close(nodeCh)
|
||||
close(errCh)
|
||||
}()
|
||||
|
||||
return nodeCh, errCh
|
||||
}
|
||||
|
||||
func (n *layer) initWorkerPool(ctx context.Context, size int, p commonVersionsListingParams, input <-chan *data.ExtendedNodeVersion) (<-chan *data.ExtendedNodeVersion, error) {
|
||||
reqLog := n.reqLogger(ctx)
|
||||
pool, err := ants.NewPool(size, ants.WithLogger(&logWrapper{reqLog}))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("coudln't init go pool for listing: %w", err)
|
||||
}
|
||||
objCh := make(chan *data.ExtendedNodeVersion, size)
|
||||
|
||||
go func() {
|
||||
var wg sync.WaitGroup
|
||||
|
||||
LOOP:
|
||||
for node := range input {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break LOOP
|
||||
default:
|
||||
}
|
||||
|
||||
if node.DirName != "" || node.NodeVersion.IsFilledExtra() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case objCh <- node:
|
||||
}
|
||||
} else {
|
||||
// We have to make a copy of pointer to data.NodeVersion
|
||||
// to get correct value in submitted task function.
|
||||
func(node *data.ExtendedNodeVersion) {
|
||||
wg.Add(1)
|
||||
err = pool.Submit(func() {
|
||||
defer wg.Done()
|
||||
|
||||
oi := n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node.NodeVersion)
|
||||
if oi == nil {
|
||||
// try to get object again
|
||||
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node.NodeVersion); oi == nil {
|
||||
// do not process object which are definitely missing in object service
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
realSize, err := GetObjectSize(oi)
|
||||
if err != nil {
|
||||
reqLog.Debug(logs.FailedToGetRealObjectSize, zap.Error(err))
|
||||
realSize = oi.Size
|
||||
}
|
||||
|
||||
node.NodeVersion.FillExtra(&oi.Owner, &oi.Created, realSize)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case objCh <- node:
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
wg.Done()
|
||||
reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err))
|
||||
}
|
||||
}(node)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
close(objCh)
|
||||
pool.Release()
|
||||
}()
|
||||
|
||||
return objCh, nil
|
||||
}
|
||||
|
||||
func shouldSkip(node *data.ExtendedNodeVersion, p commonVersionsListingParams, existed map[string]struct{}) bool {
|
||||
if node.NodeVersion.IsDeleteMarker {
|
||||
return true
|
||||
}
|
||||
|
||||
filePath := node.NodeVersion.FilePath
|
||||
if node.DirName != "" {
|
||||
filePath = node.DirName
|
||||
}
|
||||
|
||||
if _, ok := existed[filePath]; ok {
|
||||
return true
|
||||
}
|
||||
|
||||
if filePath <= p.Marker {
|
||||
return true
|
||||
}
|
||||
|
||||
if p.Bookmark != "" {
|
||||
if _, ok := existed[continuationToken]; !ok {
|
||||
if p.Bookmark != node.NodeVersion.OID.EncodeToString() {
|
||||
return true
|
||||
}
|
||||
existed[continuationToken] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
existed[filePath] = struct{}{}
|
||||
return false
|
||||
}
|
||||
|
||||
func shouldSkipVersions(node *data.ExtendedNodeVersion, p commonVersionsListingParams, existed map[string]struct{}) bool {
|
||||
filePath := node.NodeVersion.FilePath
|
||||
if node.DirName != "" {
|
||||
filePath = node.DirName
|
||||
if _, ok := existed[filePath]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if filePath < p.Marker {
|
||||
return true
|
||||
}
|
||||
|
||||
if p.Bookmark != "" {
|
||||
if _, ok := existed[continuationToken]; !ok {
|
||||
if p.Bookmark != node.NodeVersion.OID.EncodeToString() {
|
||||
return true
|
||||
}
|
||||
existed[continuationToken] = struct{}{}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
existed[filePath] = struct{}{}
|
||||
return false
|
||||
}
|
||||
|
||||
func triageExtendedObjects(allObjects []*data.ExtendedNodeVersion) (prefixes []string, objects []*data.ExtendedNodeVersion) {
|
||||
for _, ov := range allObjects {
|
||||
if ov.DirName != "" {
|
||||
prefixes = append(prefixes, ov.DirName)
|
||||
} else {
|
||||
objects = append(objects, ov)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (n *layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo *data.BucketInfo, node *data.NodeVersion) (oi *data.ObjectInfo) {
|
||||
owner := n.BearerOwner(ctx)
|
||||
if extInfo := n.cache.GetObject(owner, newAddress(bktInfo.CID, node.OID)); extInfo != nil {
|
||||
return extInfo.ObjectInfo
|
||||
}
|
||||
|
||||
meta, err := n.objectHead(ctx, bktInfo, node.OID)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err))
|
||||
return nil
|
||||
}
|
||||
|
||||
oi = objectInfoFromMeta(bktInfo, meta)
|
||||
oi.MD5Sum = node.MD5
|
||||
n.cache.PutObject(owner, &data.ExtendedObjectInfo{ObjectInfo: oi, NodeVersion: node})
|
||||
|
||||
return oi
|
||||
}
|
||||
|
||||
// tryDirectoryName forms directory name by prefix and delimiter.
|
||||
// If node isn't a directory empty string is returned.
|
||||
// This function doesn't check if node has a prefix. It must do a caller.
|
||||
func tryDirectoryName(node *data.NodeVersion, prefix, delimiter string) string {
|
||||
if len(delimiter) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
tail := strings.TrimPrefix(node.FilePath, prefix)
|
||||
index := strings.Index(tail, delimiter)
|
||||
if index >= 0 {
|
||||
return prefix + tail[:index+1]
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func filterVersionsByMarker(objects []*data.ExtendedNodeVersion, p *ListObjectVersionsParams) ([]*data.ExtendedNodeVersion, error) {
|
||||
if p.KeyMarker == "" {
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
for i, obj := range objects {
|
||||
if obj.NodeVersion.FilePath == p.KeyMarker {
|
||||
for j := i; j < len(objects); j++ {
|
||||
if objects[j].NodeVersion.FilePath != obj.NodeVersion.FilePath {
|
||||
if p.VersionIDMarker == "" {
|
||||
return objects[j:], nil
|
||||
}
|
||||
break
|
||||
}
|
||||
if objects[j].NodeVersion.OID.EncodeToString() == p.VersionIDMarker {
|
||||
return objects[j+1:], nil
|
||||
}
|
||||
}
|
||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidVersion)
|
||||
} else if obj.NodeVersion.FilePath > p.KeyMarker {
|
||||
if p.VersionIDMarker != "" {
|
||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidVersion)
|
||||
}
|
||||
return objects[i:], nil
|
||||
}
|
||||
}
|
||||
|
||||
// don't use nil as empty slice to be consistent with `return objects[j+1:], nil` above
|
||||
// that can be empty
|
||||
return []*data.ExtendedNodeVersion{}, nil
|
||||
}
|
||||
|
||||
func triageVersions(objVersions []*data.ExtendedNodeVersion) ([]*data.ExtendedNodeVersion, []*data.ExtendedNodeVersion) {
|
||||
if len(objVersions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var resVersion []*data.ExtendedNodeVersion
|
||||
var resDelMarkVersions []*data.ExtendedNodeVersion
|
||||
|
||||
for _, version := range objVersions {
|
||||
if version.NodeVersion.IsDeleteMarker {
|
||||
resDelMarkVersions = append(resDelMarkVersions, version)
|
||||
} else {
|
||||
resVersion = append(resVersion, version)
|
||||
}
|
||||
}
|
||||
|
||||
return resVersion, resDelMarkVersions
|
||||
}
|
|
@ -4,8 +4,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -20,7 +19,7 @@ func TestObjectLockAttributes(t *testing.T) {
|
|||
obj := tc.putObject([]byte("content obj1 v1"))
|
||||
|
||||
p := &PutLockInfoParams{
|
||||
ObjVersion: &data.ObjectVersion{
|
||||
ObjVersion: &ObjectVersion{
|
||||
BktInfo: tc.bktInfo,
|
||||
ObjectName: obj.Name,
|
||||
VersionID: obj.VersionID(),
|
||||
|
@ -30,7 +29,7 @@ func TestObjectLockAttributes(t *testing.T) {
|
|||
Until: time.Now(),
|
||||
},
|
||||
},
|
||||
CopiesNumbers: []uint32{0},
|
||||
CopiesNumber: 0,
|
||||
}
|
||||
|
||||
err = tc.layer.PutLockInfo(tc.ctx, p)
|
||||
|
@ -44,10 +43,10 @@ func TestObjectLockAttributes(t *testing.T) {
|
|||
|
||||
expEpoch := false
|
||||
for _, attr := range lockObj.Attributes() {
|
||||
if attr.Key() == object.SysAttributeExpEpoch {
|
||||
if attr.Key() == AttributeExpirationEpoch {
|
||||
expEpoch = true
|
||||
}
|
||||
}
|
||||
|
||||
require.Truef(t, expEpoch, "system header __SYSTEM__EXPIRATION_EPOCH presence")
|
||||
require.Truef(t, expEpoch, "system header __NEOFS__EXPIRATION_EPOCH presence")
|
||||
}
|
||||
|
|
|
@ -1,156 +0,0 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
type partObj struct {
|
||||
OID oid.ID
|
||||
Size uint64
|
||||
}
|
||||
|
||||
type readerInitiator interface {
|
||||
initFrostFSObjectPayloadReader(ctx context.Context, p getFrostFSParams) (io.Reader, error)
|
||||
}
|
||||
|
||||
// implements io.Reader of payloads of the object list stored in the FrostFS network.
|
||||
type multiObjectReader struct {
|
||||
ctx context.Context
|
||||
|
||||
layer readerInitiator
|
||||
|
||||
startPartOffset uint64
|
||||
endPartLength uint64
|
||||
|
||||
prm getFrostFSParams
|
||||
|
||||
curIndex int
|
||||
curReader io.Reader
|
||||
|
||||
parts []partObj
|
||||
}
|
||||
|
||||
type multiObjectReaderConfig struct {
|
||||
layer readerInitiator
|
||||
|
||||
// the offset of complete object and total size to read
|
||||
off, ln uint64
|
||||
|
||||
bktInfo *data.BucketInfo
|
||||
parts []partObj
|
||||
}
|
||||
|
||||
var (
|
||||
errOffsetIsOutOfRange = errors.New("offset is out of payload range")
|
||||
errLengthIsOutOfRange = errors.New("length is out of payload range")
|
||||
errEmptyPartsList = errors.New("empty parts list")
|
||||
errorZeroRangeLength = errors.New("zero range length")
|
||||
)
|
||||
|
||||
func newMultiObjectReader(ctx context.Context, cfg multiObjectReaderConfig) (*multiObjectReader, error) {
|
||||
if len(cfg.parts) == 0 {
|
||||
return nil, errEmptyPartsList
|
||||
}
|
||||
|
||||
r := &multiObjectReader{
|
||||
ctx: ctx,
|
||||
layer: cfg.layer,
|
||||
prm: getFrostFSParams{
|
||||
bktInfo: cfg.bktInfo,
|
||||
},
|
||||
parts: cfg.parts,
|
||||
}
|
||||
|
||||
if cfg.off+cfg.ln == 0 {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
if cfg.off > 0 && cfg.ln == 0 {
|
||||
return nil, errorZeroRangeLength
|
||||
}
|
||||
|
||||
startPartIndex, startPartOffset := findStartPart(cfg)
|
||||
if startPartIndex == -1 {
|
||||
return nil, errOffsetIsOutOfRange
|
||||
}
|
||||
r.startPartOffset = startPartOffset
|
||||
|
||||
endPartIndex, endPartLength := findEndPart(cfg)
|
||||
if endPartIndex == -1 {
|
||||
return nil, errLengthIsOutOfRange
|
||||
}
|
||||
r.endPartLength = endPartLength
|
||||
|
||||
r.parts = cfg.parts[startPartIndex : endPartIndex+1]
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func findStartPart(cfg multiObjectReaderConfig) (index int, offset uint64) {
|
||||
position := cfg.off
|
||||
for i, part := range cfg.parts {
|
||||
// Strict inequality when searching for start position to avoid reading zero length part.
|
||||
if position < part.Size {
|
||||
return i, position
|
||||
}
|
||||
position -= part.Size
|
||||
}
|
||||
|
||||
return -1, 0
|
||||
}
|
||||
|
||||
func findEndPart(cfg multiObjectReaderConfig) (index int, length uint64) {
|
||||
position := cfg.off + cfg.ln
|
||||
for i, part := range cfg.parts {
|
||||
// Non-strict inequality when searching for end position to avoid out of payload range error.
|
||||
if position <= part.Size {
|
||||
return i, position
|
||||
}
|
||||
position -= part.Size
|
||||
}
|
||||
|
||||
return -1, 0
|
||||
}
|
||||
|
||||
func (x *multiObjectReader) Read(p []byte) (n int, err error) {
|
||||
if x.curReader != nil {
|
||||
n, err = x.curReader.Read(p)
|
||||
if !errors.Is(err, io.EOF) {
|
||||
return n, err
|
||||
}
|
||||
x.curIndex++
|
||||
}
|
||||
|
||||
if x.curIndex == len(x.parts) {
|
||||
return n, io.EOF
|
||||
}
|
||||
|
||||
x.prm.oid = x.parts[x.curIndex].OID
|
||||
|
||||
if x.curIndex == 0 {
|
||||
x.prm.off = x.startPartOffset
|
||||
x.prm.ln = x.parts[x.curIndex].Size - x.startPartOffset
|
||||
}
|
||||
|
||||
if x.curIndex == len(x.parts)-1 {
|
||||
x.prm.ln = x.endPartLength - x.prm.off
|
||||
}
|
||||
|
||||
x.curReader, err = x.layer.initFrostFSObjectPayloadReader(x.ctx, x.prm)
|
||||
if err != nil {
|
||||
return n, fmt.Errorf("init payload reader for the next part: %w", err)
|
||||
}
|
||||
|
||||
x.prm.off = 0
|
||||
x.prm.ln = 0
|
||||
|
||||
next, err := x.Read(p[n:])
|
||||
|
||||
return n + next, err
|
||||
}
|
|
@ -1,137 +0,0 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type readerInitiatorMock struct {
|
||||
parts map[oid.ID][]byte
|
||||
}
|
||||
|
||||
func (r *readerInitiatorMock) initFrostFSObjectPayloadReader(_ context.Context, p getFrostFSParams) (io.Reader, error) {
|
||||
partPayload, ok := r.parts[p.oid]
|
||||
if !ok {
|
||||
return nil, errors.New("part not found")
|
||||
}
|
||||
|
||||
if p.off+p.ln == 0 {
|
||||
return bytes.NewReader(partPayload), nil
|
||||
}
|
||||
|
||||
if p.off > uint64(len(partPayload)-1) {
|
||||
return nil, fmt.Errorf("invalid offset: %d/%d", p.off, len(partPayload))
|
||||
}
|
||||
|
||||
if p.off+p.ln > uint64(len(partPayload)) {
|
||||
return nil, fmt.Errorf("invalid range: %d-%d/%d", p.off, p.off+p.ln, len(partPayload))
|
||||
}
|
||||
|
||||
return bytes.NewReader(partPayload[p.off : p.off+p.ln]), nil
|
||||
}
|
||||
|
||||
func prepareDataReader() ([]byte, []partObj, *readerInitiatorMock) {
|
||||
mockInitReader := &readerInitiatorMock{
|
||||
parts: map[oid.ID][]byte{
|
||||
oidtest.ID(): []byte("first part 1"),
|
||||
oidtest.ID(): []byte("second part 2"),
|
||||
oidtest.ID(): []byte("third part 3"),
|
||||
},
|
||||
}
|
||||
|
||||
var fullPayload []byte
|
||||
parts := make([]partObj, 0, len(mockInitReader.parts))
|
||||
for id, payload := range mockInitReader.parts {
|
||||
parts = append(parts, partObj{OID: id, Size: uint64(len(payload))})
|
||||
fullPayload = append(fullPayload, payload...)
|
||||
}
|
||||
|
||||
return fullPayload, parts, mockInitReader
|
||||
}
|
||||
|
||||
func TestMultiReader(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
fullPayload, parts, mockInitReader := prepareDataReader()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
off uint64
|
||||
ln uint64
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "simple read all",
|
||||
},
|
||||
{
|
||||
name: "simple read with length",
|
||||
ln: uint64(len(fullPayload)),
|
||||
},
|
||||
{
|
||||
name: "middle of parts",
|
||||
off: parts[0].Size + 2,
|
||||
ln: 4,
|
||||
},
|
||||
{
|
||||
name: "first and second",
|
||||
off: parts[0].Size - 4,
|
||||
ln: 8,
|
||||
},
|
||||
{
|
||||
name: "first and third",
|
||||
off: parts[0].Size - 4,
|
||||
ln: parts[1].Size + 8,
|
||||
},
|
||||
{
|
||||
name: "second part",
|
||||
off: parts[0].Size,
|
||||
ln: parts[1].Size,
|
||||
},
|
||||
{
|
||||
name: "second and third",
|
||||
off: parts[0].Size,
|
||||
ln: parts[1].Size + parts[2].Size,
|
||||
},
|
||||
{
|
||||
name: "offset out of range",
|
||||
off: uint64(len(fullPayload) + 1),
|
||||
ln: 1,
|
||||
err: errOffsetIsOutOfRange,
|
||||
},
|
||||
{
|
||||
name: "zero length",
|
||||
off: parts[1].Size + 1,
|
||||
ln: 0,
|
||||
err: errorZeroRangeLength,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
multiReader, err := newMultiObjectReader(ctx, multiObjectReaderConfig{
|
||||
layer: mockInitReader,
|
||||
parts: parts,
|
||||
off: tc.off,
|
||||
ln: tc.ln,
|
||||
})
|
||||
require.ErrorIs(t, err, tc.err)
|
||||
|
||||
if tc.err == nil {
|
||||
off := tc.off
|
||||
ln := tc.ln
|
||||
if off+ln == 0 {
|
||||
ln = uint64(len(fullPayload))
|
||||
}
|
||||
data, err := io.ReadAll(multiReader)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, fullPayload[off:off+ln], data)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue