[#142] Fix multipart-objects download
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
parent
843708a558
commit
33dba5d68c
16 changed files with 803 additions and 230 deletions
|
@ -17,11 +17,10 @@ import (
|
||||||
|
|
||||||
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
|
@ -400,7 +399,7 @@ func (a *app) setHealthStatus() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *app) Serve() {
|
func (a *app) Serve() {
|
||||||
handler := handler.New(a.AppParams(), a.settings, tree.NewTree(services.NewPoolWrapper(a.treePool)))
|
handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)))
|
||||||
|
|
||||||
// Configure router.
|
// Configure router.
|
||||||
a.configureRouter(handler)
|
a.configureRouter(handler)
|
||||||
|
|
26
go.mod
26
go.mod
|
@ -9,7 +9,8 @@ require (
|
||||||
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
|
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
|
||||||
github.com/bluele/gcache v0.0.2
|
github.com/bluele/gcache v0.0.2
|
||||||
github.com/fasthttp/router v1.4.1
|
github.com/fasthttp/router v1.4.1
|
||||||
github.com/nspcc-dev/neo-go v0.106.2
|
github.com/minio/sio v0.4.1
|
||||||
|
github.com/nspcc-dev/neo-go v0.106.3
|
||||||
github.com/prometheus/client_golang v1.19.0
|
github.com/prometheus/client_golang v1.19.0
|
||||||
github.com/prometheus/client_model v0.5.0
|
github.com/prometheus/client_model v0.5.0
|
||||||
github.com/spf13/pflag v1.0.5
|
github.com/spf13/pflag v1.0.5
|
||||||
|
@ -28,7 +29,7 @@ require (
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e // indirect
|
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.4 // indirect
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
|
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
|
||||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect
|
git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect
|
||||||
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
|
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
|
||||||
|
@ -43,7 +44,7 @@ require (
|
||||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||||
github.com/containerd/cgroups v1.0.3 // indirect
|
github.com/containerd/cgroups v1.0.3 // indirect
|
||||||
github.com/containerd/containerd v1.6.2 // indirect
|
github.com/containerd/containerd v1.6.2 // indirect
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
|
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
|
||||||
github.com/docker/distribution v2.8.1+incompatible // indirect
|
github.com/docker/distribution v2.8.1+incompatible // indirect
|
||||||
|
@ -56,13 +57,14 @@ require (
|
||||||
github.com/gogo/protobuf v1.3.2 // indirect
|
github.com/gogo/protobuf v1.3.2 // indirect
|
||||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||||
github.com/golang/protobuf v1.5.3 // indirect
|
github.com/golang/protobuf v1.5.3 // indirect
|
||||||
github.com/golang/snappy v0.0.4 // indirect
|
github.com/golang/snappy v0.0.3 // indirect
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
github.com/gorilla/mux v1.8.0 // indirect
|
github.com/gorilla/mux v1.8.0 // indirect
|
||||||
github.com/gorilla/websocket v1.5.1 // indirect
|
github.com/gorilla/websocket v1.5.1 // indirect
|
||||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
|
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||||
|
github.com/holiman/uint256 v1.2.4 // indirect
|
||||||
github.com/klauspost/compress v1.16.4 // indirect
|
github.com/klauspost/compress v1.16.4 // indirect
|
||||||
github.com/magiconair/properties v1.8.7 // indirect
|
github.com/magiconair/properties v1.8.7 // indirect
|
||||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||||
|
@ -72,12 +74,13 @@ require (
|
||||||
github.com/morikuni/aec v1.0.0 // indirect
|
github.com/morikuni/aec v1.0.0 // indirect
|
||||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect
|
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect
|
||||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect
|
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240727093519-1a48f1ce43ec // indirect
|
||||||
github.com/nspcc-dev/rfc6979 v0.2.1 // indirect
|
github.com/nspcc-dev/rfc6979 v0.2.1 // indirect
|
||||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||||
github.com/opencontainers/image-spec v1.0.2 // indirect
|
github.com/opencontainers/image-spec v1.0.2 // indirect
|
||||||
github.com/opencontainers/runc v1.1.1 // indirect
|
github.com/opencontainers/runc v1.1.1 // indirect
|
||||||
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
|
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
|
||||||
|
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
|
||||||
github.com/pkg/errors v0.9.1 // indirect
|
github.com/pkg/errors v0.9.1 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
github.com/prometheus/common v0.48.0 // indirect
|
github.com/prometheus/common v0.48.0 // indirect
|
||||||
|
@ -89,10 +92,11 @@ require (
|
||||||
github.com/spf13/cast v1.5.0 // indirect
|
github.com/spf13/cast v1.5.0 // indirect
|
||||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||||
github.com/subosito/gotenv v1.4.2 // indirect
|
github.com/subosito/gotenv v1.4.2 // indirect
|
||||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
|
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect
|
||||||
github.com/twmb/murmur3 v1.1.8 // indirect
|
github.com/twmb/murmur3 v1.1.8 // indirect
|
||||||
github.com/urfave/cli v1.22.5 // indirect
|
github.com/urfave/cli/v2 v2.27.2 // indirect
|
||||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||||
|
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
|
||||||
go.etcd.io/bbolt v1.3.9 // indirect
|
go.etcd.io/bbolt v1.3.9 // indirect
|
||||||
go.opencensus.io v0.24.0 // indirect
|
go.opencensus.io v0.24.0 // indirect
|
||||||
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
|
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
|
||||||
|
@ -103,11 +107,11 @@ require (
|
||||||
go.opentelemetry.io/otel/sdk v1.16.0 // indirect
|
go.opentelemetry.io/otel/sdk v1.16.0 // indirect
|
||||||
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
|
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
|
||||||
go.uber.org/multierr v1.11.0 // indirect
|
go.uber.org/multierr v1.11.0 // indirect
|
||||||
golang.org/x/crypto v0.21.0 // indirect
|
golang.org/x/crypto v0.23.0 // indirect
|
||||||
golang.org/x/sync v0.6.0 // indirect
|
golang.org/x/sync v0.6.0 // indirect
|
||||||
golang.org/x/sys v0.18.0 // indirect
|
golang.org/x/sys v0.20.0 // indirect
|
||||||
golang.org/x/term v0.18.0 // indirect
|
golang.org/x/term v0.20.0 // indirect
|
||||||
golang.org/x/text v0.14.0 // indirect
|
golang.org/x/text v0.15.0 // indirect
|
||||||
golang.org/x/time v0.3.0 // indirect
|
golang.org/x/time v0.3.0 // indirect
|
||||||
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
|
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
|
||||||
google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect
|
google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect
|
||||||
|
|
52
go.sum
52
go.sum
|
@ -39,8 +39,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
|
||||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 h1:XxvwQKJT/f16qS3df5PBQPRYKkhy0/A7zH6644QpKD0=
|
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 h1:XxvwQKJT/f16qS3df5PBQPRYKkhy0/A7zH6644QpKD0=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o=
|
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4=
|
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.4 h1:HYukyt9WGKBnkexnIlzg6Am3SvH48QZbtIxt0YeObBs=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc=
|
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.4/go.mod h1:Y2Xorxc8SBO4phoek7n3XxaPZz5rIrFgDsU4TOjmlGA=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
|
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
|
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
|
||||||
|
@ -280,8 +280,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
|
||||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||||
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
|
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
|
||||||
|
@ -438,9 +438,9 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
|
||||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||||
|
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||||
|
github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
|
||||||
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
|
||||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
|
||||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||||
|
@ -587,6 +587,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
|
||||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||||
github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
|
github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
|
||||||
github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
|
github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
|
||||||
|
github.com/minio/sio v0.4.1 h1:EMe3YBC1nf+sRQia65Rutxi+Z554XPV0dt8BIBA+a/0=
|
||||||
|
github.com/minio/sio v0.4.1/go.mod h1:oBSjJeGbBdRMZZwna07sX9EFzZy+ywu5aofRiV1g79I=
|
||||||
github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
|
github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
|
||||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||||
|
@ -624,12 +626,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
|
||||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
|
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
|
||||||
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
|
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
|
||||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||||
|
github.com/nspcc-dev/dbft v0.2.0 h1:sDwsQES600OSIMncV176t2SX5OvB14lzeOAyKFOkbMI=
|
||||||
|
github.com/nspcc-dev/dbft v0.2.0/go.mod h1:oFE6paSC/yfFh9mcNU6MheMGOYXK9+sPiRk3YMoz49o=
|
||||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk=
|
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk=
|
||||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc=
|
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc=
|
||||||
github.com/nspcc-dev/neo-go v0.106.2 h1:KXSJ2J5Oacc7LrX3r4jvnC8ihKqHs5NB21q4f2S3r9o=
|
github.com/nspcc-dev/neo-go v0.106.3 h1:HEyhgkjQY+HfBzotMJ12xx2VuOUphkngZ4kEkjvXDtE=
|
||||||
github.com/nspcc-dev/neo-go v0.106.2/go.mod h1:Ojwfx3/lv0VTeEHMpQ17g0wTnXcCSoFQVq5GEeCZmGo=
|
github.com/nspcc-dev/neo-go v0.106.3/go.mod h1:3vEwJ2ld12N7HRGCaH/l/7EwopplC/+8XdIdPDNmD/M=
|
||||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d h1:Vcb7YkZuUSSIC+WF/xV3UDfHbAxZgyT2zGleJP3Ig5k=
|
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240727093519-1a48f1ce43ec h1:vDrbVXF2+2uP0RlkZmem3QYATcXCu9BzzGGCNsNcK7Q=
|
||||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d/go.mod h1:/vrbWSHc7YS1KSYhVOyyeucXW/e+1DkVBOgnBEXUCeY=
|
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240727093519-1a48f1ce43ec/go.mod h1:/vrbWSHc7YS1KSYhVOyyeucXW/e+1DkVBOgnBEXUCeY=
|
||||||
github.com/nspcc-dev/rfc6979 v0.2.1 h1:8wWxkamHWFmO790GsewSoKUSJjVnL1fmdRpokU/RgRM=
|
github.com/nspcc-dev/rfc6979 v0.2.1 h1:8wWxkamHWFmO790GsewSoKUSJjVnL1fmdRpokU/RgRM=
|
||||||
github.com/nspcc-dev/rfc6979 v0.2.1/go.mod h1:Tk7h5kyUWkhjyO3zUgFFhy1v2vQv3BvQEntakdtqrWc=
|
github.com/nspcc-dev/rfc6979 v0.2.1/go.mod h1:Tk7h5kyUWkhjyO3zUgFFhy1v2vQv3BvQEntakdtqrWc=
|
||||||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
||||||
|
@ -690,6 +694,8 @@ github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrap
|
||||||
github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
|
github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
|
||||||
github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
|
github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
|
||||||
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
|
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
|
||||||
|
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
|
||||||
|
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
|
@ -813,8 +819,8 @@ github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNG
|
||||||
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
|
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
|
||||||
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
|
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
|
||||||
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
|
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
|
||||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
|
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs=
|
||||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
|
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM=
|
||||||
github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
|
github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
|
||||||
github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBNGN0TYb/7oKIPVn15JA=
|
github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBNGN0TYb/7oKIPVn15JA=
|
||||||
github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk=
|
github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk=
|
||||||
|
@ -829,8 +835,8 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/
|
||||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||||
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||||
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||||
github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
|
github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
|
||||||
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
|
||||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||||
github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA=
|
github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA=
|
||||||
|
@ -850,6 +856,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
|
||||||
github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
|
github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
|
||||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||||
|
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
|
||||||
|
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
|
||||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
|
@ -921,8 +929,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
|
||||||
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
|
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
|
||||||
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||||
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
|
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
|
||||||
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
|
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||||
|
@ -1127,12 +1135,12 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
|
||||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
|
golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
|
||||||
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
|
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
@ -1142,8 +1150,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
|
||||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
|
|
41
internal/data/object.go
Normal file
41
internal/data/object.go
Normal file
|
@ -0,0 +1,41 @@
|
||||||
|
package data
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
ObjectInfo struct {
|
||||||
|
ID oid.ID
|
||||||
|
CID cid.ID
|
||||||
|
|
||||||
|
Bucket string
|
||||||
|
Name string
|
||||||
|
Size uint64
|
||||||
|
Headers map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PartInfo is upload information about part.
|
||||||
|
PartInfo struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
UploadID string `json:"uploadId"`
|
||||||
|
Number int `json:"number"`
|
||||||
|
OID oid.ID `json:"oid"`
|
||||||
|
Size uint64 `json:"size"`
|
||||||
|
ETag string `json:"etag"`
|
||||||
|
MD5 string `json:"md5"`
|
||||||
|
Created time.Time `json:"created"`
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// Address returns object address.
|
||||||
|
func (o *ObjectInfo) Address() oid.Address {
|
||||||
|
var addr oid.Address
|
||||||
|
addr.SetContainer(o.CID)
|
||||||
|
addr.SetObject(o.ID)
|
||||||
|
|
||||||
|
return addr
|
||||||
|
}
|
|
@ -11,6 +11,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
|
@ -45,13 +46,13 @@ func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) {
|
||||||
h.byAttribute(c, h.receiveFile)
|
h.byAttribute(c, h.receiveFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) {
|
func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op object.SearchMatchType) (frostfs.ResObjectSearch, error) {
|
||||||
filters := object.NewSearchFilters()
|
filters := object.NewSearchFilters()
|
||||||
filters.AddRootFilter()
|
filters.AddRootFilter()
|
||||||
filters.AddFilter(key, val, op)
|
filters.AddFilter(key, val, op)
|
||||||
|
|
||||||
prm := PrmObjectSearch{
|
prm := frostfs.PrmObjectSearch{
|
||||||
PrmAuth: PrmAuth{
|
PrmAuth: frostfs.PrmAuth{
|
||||||
BearerToken: bearerToken(ctx),
|
BearerToken: bearerToken(ctx),
|
||||||
},
|
},
|
||||||
Container: *cnrID,
|
Container: *cnrID,
|
||||||
|
@ -153,8 +154,8 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error {
|
func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error {
|
||||||
prm := PrmObjectGet{
|
prm := frostfs.PrmObjectGet{
|
||||||
PrmAuth: PrmAuth{
|
PrmAuth: frostfs.PrmAuth{
|
||||||
BearerToken: btoken,
|
BearerToken: btoken,
|
||||||
},
|
},
|
||||||
Address: addr,
|
Address: addr,
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
|
||||||
|
@ -58,7 +59,7 @@ func (t *TestFrostFS) AllowUserOperation(cnrID cid.ID, userID user.ID, op acl.Op
|
||||||
t.accessList[fmt.Sprintf("%s/%s/%s/%s", cnrID, userID, op, objID)] = true
|
t.accessList[fmt.Sprintf("%s/%s/%s/%s", cnrID, userID, op, objID)] = true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) Container(_ context.Context, prm PrmContainer) (*container.Container, error) {
|
func (t *TestFrostFS) Container(_ context.Context, prm frostfs.PrmContainer) (*container.Container, error) {
|
||||||
for k, v := range t.containers {
|
for k, v := range t.containers {
|
||||||
if k == prm.ContainerID.EncodeToString() {
|
if k == prm.ContainerID.EncodeToString() {
|
||||||
return v, nil
|
return v, nil
|
||||||
|
@ -85,7 +86,7 @@ func (t *TestFrostFS) retrieveObject(addr oid.Address, btoken *bearer.Token) (*o
|
||||||
owner := t.requestOwner(btoken)
|
owner := t.requestOwner(btoken)
|
||||||
|
|
||||||
if !t.isAllowed(addr.Container(), owner, acl.OpObjectGet, addr.Object()) {
|
if !t.isAllowed(addr.Container(), owner, acl.OpObjectGet, addr.Object()) {
|
||||||
return nil, ErrAccessDenied
|
return nil, frostfs.ErrAccessDenied
|
||||||
}
|
}
|
||||||
|
|
||||||
return obj, nil
|
return obj, nil
|
||||||
|
@ -94,23 +95,23 @@ func (t *TestFrostFS) retrieveObject(addr oid.Address, btoken *bearer.Token) (*o
|
||||||
return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, addr)
|
return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) HeadObject(_ context.Context, prm PrmObjectHead) (*object.Object, error) {
|
func (t *TestFrostFS) HeadObject(_ context.Context, prm frostfs.PrmObjectHead) (*object.Object, error) {
|
||||||
return t.retrieveObject(prm.Address, prm.BearerToken)
|
return t.retrieveObject(prm.Address, prm.BearerToken)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) GetObject(_ context.Context, prm PrmObjectGet) (*Object, error) {
|
func (t *TestFrostFS) GetObject(_ context.Context, prm frostfs.PrmObjectGet) (*frostfs.Object, error) {
|
||||||
obj, err := t.retrieveObject(prm.Address, prm.BearerToken)
|
obj, err := t.retrieveObject(prm.Address, prm.BearerToken)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return &Object{
|
return &frostfs.Object{
|
||||||
Header: *obj,
|
Header: *obj,
|
||||||
Payload: io.NopCloser(bytes.NewReader(obj.Payload())),
|
Payload: io.NopCloser(bytes.NewReader(obj.Payload())),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) RangeObject(_ context.Context, prm PrmObjectRange) (io.ReadCloser, error) {
|
func (t *TestFrostFS) RangeObject(_ context.Context, prm frostfs.PrmObjectRange) (io.ReadCloser, error) {
|
||||||
obj, err := t.retrieveObject(prm.Address, prm.BearerToken)
|
obj, err := t.retrieveObject(prm.Address, prm.BearerToken)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -121,7 +122,7 @@ func (t *TestFrostFS) RangeObject(_ context.Context, prm PrmObjectRange) (io.Rea
|
||||||
return io.NopCloser(bytes.NewReader(payload)), nil
|
return io.NopCloser(bytes.NewReader(payload)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID, error) {
|
func (t *TestFrostFS) CreateObject(_ context.Context, prm frostfs.PrmObjectCreate) (oid.ID, error) {
|
||||||
b := make([]byte, 32)
|
b := make([]byte, 32)
|
||||||
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
||||||
return oid.ID{}, err
|
return oid.ID{}, err
|
||||||
|
@ -158,7 +159,7 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.
|
||||||
owner := t.requestOwner(prm.BearerToken)
|
owner := t.requestOwner(prm.BearerToken)
|
||||||
|
|
||||||
if !t.isAllowed(cnrID, owner, acl.OpObjectPut, objID) {
|
if !t.isAllowed(cnrID, owner, acl.OpObjectPut, objID) {
|
||||||
return oid.ID{}, ErrAccessDenied
|
return oid.ID{}, frostfs.ErrAccessDenied
|
||||||
}
|
}
|
||||||
|
|
||||||
addr := newAddress(cnrID, objID)
|
addr := newAddress(cnrID, objID)
|
||||||
|
@ -195,9 +196,9 @@ func (r *resObjectSearchMock) Iterate(f func(oid.ID) bool) error {
|
||||||
|
|
||||||
func (r *resObjectSearchMock) Close() {}
|
func (r *resObjectSearchMock) Close() {}
|
||||||
|
|
||||||
func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) (ResObjectSearch, error) {
|
func (t *TestFrostFS) SearchObjects(_ context.Context, prm frostfs.PrmObjectSearch) (frostfs.ResObjectSearch, error) {
|
||||||
if !t.isAllowed(prm.Container, t.requestOwner(prm.BearerToken), acl.OpObjectSearch, oid.ID{}) {
|
if !t.isAllowed(prm.Container, t.requestOwner(prm.BearerToken), acl.OpObjectSearch, oid.ID{}) {
|
||||||
return nil, ErrAccessDenied
|
return nil, frostfs.ErrAccessDenied
|
||||||
}
|
}
|
||||||
|
|
||||||
cidStr := prm.Container.EncodeToString()
|
cidStr := prm.Container.EncodeToString()
|
||||||
|
|
|
@ -12,10 +12,10 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
|
||||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
@ -34,110 +34,14 @@ type Config interface {
|
||||||
NamespaceHeader() string
|
NamespaceHeader() string
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrmContainer groups parameters of FrostFS.Container operation.
|
|
||||||
type PrmContainer struct {
|
|
||||||
// Container identifier.
|
|
||||||
ContainerID cid.ID
|
|
||||||
}
|
|
||||||
|
|
||||||
// PrmAuth groups authentication parameters for the FrostFS operation.
|
|
||||||
type PrmAuth struct {
|
|
||||||
// Bearer token to be used for the operation. Overlaps PrivateKey. Optional.
|
|
||||||
BearerToken *bearer.Token
|
|
||||||
}
|
|
||||||
|
|
||||||
// PrmObjectHead groups parameters of FrostFS.HeadObject operation.
|
|
||||||
type PrmObjectHead struct {
|
|
||||||
// Authentication parameters.
|
|
||||||
PrmAuth
|
|
||||||
|
|
||||||
// Address to read the object header from.
|
|
||||||
Address oid.Address
|
|
||||||
}
|
|
||||||
|
|
||||||
// PrmObjectGet groups parameters of FrostFS.GetObject operation.
|
|
||||||
type PrmObjectGet struct {
|
|
||||||
// Authentication parameters.
|
|
||||||
PrmAuth
|
|
||||||
|
|
||||||
// Address to read the object header from.
|
|
||||||
Address oid.Address
|
|
||||||
}
|
|
||||||
|
|
||||||
// PrmObjectRange groups parameters of FrostFS.RangeObject operation.
|
|
||||||
type PrmObjectRange struct {
|
|
||||||
// Authentication parameters.
|
|
||||||
PrmAuth
|
|
||||||
|
|
||||||
// Address to read the object header from.
|
|
||||||
Address oid.Address
|
|
||||||
|
|
||||||
// Offset-length range of the object payload to be read.
|
|
||||||
PayloadRange [2]uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// Object represents FrostFS object.
|
|
||||||
type Object struct {
|
|
||||||
// Object header (doesn't contain payload).
|
|
||||||
Header object.Object
|
|
||||||
|
|
||||||
// Object payload part encapsulated in io.Reader primitive.
|
|
||||||
// Returns ErrAccessDenied on read access violation.
|
|
||||||
Payload io.ReadCloser
|
|
||||||
}
|
|
||||||
|
|
||||||
// PrmObjectCreate groups parameters of FrostFS.CreateObject operation.
|
|
||||||
type PrmObjectCreate struct {
|
|
||||||
// Authentication parameters.
|
|
||||||
PrmAuth
|
|
||||||
|
|
||||||
Object *object.Object
|
|
||||||
|
|
||||||
// Object payload encapsulated in io.Reader primitive.
|
|
||||||
Payload io.Reader
|
|
||||||
|
|
||||||
// Enables client side object preparing.
|
|
||||||
ClientCut bool
|
|
||||||
|
|
||||||
// Disables using Tillich-Zémor hash for payload.
|
|
||||||
WithoutHomomorphicHash bool
|
|
||||||
|
|
||||||
// Sets max buffer size to read payload.
|
|
||||||
BufferMaxSize uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// PrmObjectSearch groups parameters of FrostFS.sear SearchObjects operation.
|
|
||||||
type PrmObjectSearch struct {
|
|
||||||
// Authentication parameters.
|
|
||||||
PrmAuth
|
|
||||||
|
|
||||||
// Container to select the objects from.
|
|
||||||
Container cid.ID
|
|
||||||
|
|
||||||
Filters object.SearchFilters
|
|
||||||
}
|
|
||||||
|
|
||||||
type ResObjectSearch interface {
|
|
||||||
Read(buf []oid.ID) (int, error)
|
|
||||||
Iterate(f func(oid.ID) bool) error
|
|
||||||
Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
// ErrAccessDenied is returned from FrostFS in case of access violation.
|
|
||||||
ErrAccessDenied = errors.New("access denied")
|
|
||||||
// ErrGatewayTimeout is returned from FrostFS in case of timeout, deadline exceeded etc.
|
|
||||||
ErrGatewayTimeout = errors.New("gateway timeout")
|
|
||||||
)
|
|
||||||
|
|
||||||
// FrostFS represents virtual connection to FrostFS network.
|
// FrostFS represents virtual connection to FrostFS network.
|
||||||
type FrostFS interface {
|
type FrostFS interface {
|
||||||
Container(context.Context, PrmContainer) (*container.Container, error)
|
Container(context.Context, frostfs.PrmContainer) (*container.Container, error)
|
||||||
HeadObject(context.Context, PrmObjectHead) (*object.Object, error)
|
HeadObject(context.Context, frostfs.PrmObjectHead) (*object.Object, error)
|
||||||
GetObject(context.Context, PrmObjectGet) (*Object, error)
|
GetObject(context.Context, frostfs.PrmObjectGet) (*frostfs.Object, error)
|
||||||
RangeObject(context.Context, PrmObjectRange) (io.ReadCloser, error)
|
RangeObject(context.Context, frostfs.PrmObjectRange) (io.ReadCloser, error)
|
||||||
CreateObject(context.Context, PrmObjectCreate) (oid.ID, error)
|
CreateObject(context.Context, frostfs.PrmObjectCreate) (oid.ID, error)
|
||||||
SearchObjects(context.Context, PrmObjectSearch) (ResObjectSearch, error)
|
SearchObjects(context.Context, frostfs.PrmObjectSearch) (frostfs.ResObjectSearch, error)
|
||||||
utils.EpochInfoFetcher
|
utils.EpochInfoFetcher
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -177,7 +81,7 @@ func New(params *AppParams, config Config, tree *tree.Tree) *Handler {
|
||||||
|
|
||||||
// byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that
|
// byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that
|
||||||
// prepares request and object address to it.
|
// prepares request and object address to it.
|
||||||
func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address, *data.BucketInfo)) {
|
||||||
var (
|
var (
|
||||||
idCnr, _ = c.UserValue("cid").(string)
|
idCnr, _ = c.UserValue("cid").(string)
|
||||||
idObj, _ = c.UserValue("oid").(string)
|
idObj, _ = c.UserValue("oid").(string)
|
||||||
|
@ -203,12 +107,12 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ
|
||||||
addr.SetContainer(bktInfo.CID)
|
addr.SetContainer(bktInfo.CID)
|
||||||
addr.SetObject(*objID)
|
addr.SetObject(*objID)
|
||||||
|
|
||||||
f(ctx, *h.newRequest(c, log), addr)
|
f(ctx, *h.newRequest(c, log), addr, bktInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
// byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that
|
// byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that
|
||||||
// prepares request and object address to it.
|
// prepares request and object address to it.
|
||||||
func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address, *data.BucketInfo)) {
|
||||||
var (
|
var (
|
||||||
bucketname = req.UserValue("cid").(string)
|
bucketname = req.UserValue("cid").(string)
|
||||||
key = req.UserValue("oid").(string)
|
key = req.UserValue("oid").(string)
|
||||||
|
@ -250,11 +154,11 @@ func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context,
|
||||||
addr.SetContainer(bktInfo.CID)
|
addr.SetContainer(bktInfo.CID)
|
||||||
addr.SetObject(foundOid.OID)
|
addr.SetObject(foundOid.OID)
|
||||||
|
|
||||||
f(ctx, *h.newRequest(req, log), addr)
|
f(ctx, *h.newRequest(req, log), addr, bktInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
// byAttribute is a wrapper similar to byAddress.
|
// byAttribute is a wrapper similar to byAddress.
|
||||||
func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address, *data.BucketInfo)) {
|
||||||
scid, _ := c.UserValue("cid").(string)
|
scid, _ := c.UserValue("cid").(string)
|
||||||
key, _ := c.UserValue("attr_key").(string)
|
key, _ := c.UserValue("attr_key").(string)
|
||||||
val, _ := c.UserValue("attr_val").(string)
|
val, _ := c.UserValue("attr_val").(string)
|
||||||
|
@ -311,7 +215,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re
|
||||||
addrObj.SetContainer(bktInfo.CID)
|
addrObj.SetContainer(bktInfo.CID)
|
||||||
addrObj.SetObject(buf[0])
|
addrObj.SetObject(buf[0])
|
||||||
|
|
||||||
f(ctx, *h.newRequest(c, log), addrObj)
|
f(ctx, *h.newRequest(c, log), addrObj, bktInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveContainer decode container id, if it's not a valid container id
|
// resolveContainer decode container id, if it's not a valid container id
|
||||||
|
@ -359,7 +263,7 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log *
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) {
|
func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) {
|
||||||
prm := PrmContainer{ContainerID: cnrID}
|
prm := frostfs.PrmContainer{ContainerID: cnrID}
|
||||||
res, err := h.frostfs.Container(ctx, prm)
|
res, err := h.frostfs.Container(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("get frostfs container '%s': %w", cnrID.String(), err)
|
return nil, fmt.Errorf("get frostfs container '%s': %w", cnrID.String(), err)
|
||||||
|
|
|
@ -7,7 +7,9 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
@ -24,13 +26,13 @@ const (
|
||||||
hdrContainerID = "X-Container-Id"
|
hdrContainerID = "X-Container-Id"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid.Address) {
|
func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid.Address, _ *data.BucketInfo) {
|
||||||
var start = time.Now()
|
var start = time.Now()
|
||||||
|
|
||||||
btoken := bearerToken(ctx)
|
btoken := bearerToken(ctx)
|
||||||
|
|
||||||
prm := PrmObjectHead{
|
prm := frostfs.PrmObjectHead{
|
||||||
PrmAuth: PrmAuth{
|
PrmAuth: frostfs.PrmAuth{
|
||||||
BearerToken: btoken,
|
BearerToken: btoken,
|
||||||
},
|
},
|
||||||
Address: objectAddress,
|
Address: objectAddress,
|
||||||
|
@ -74,8 +76,8 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid
|
||||||
|
|
||||||
if len(contentType) == 0 {
|
if len(contentType) == 0 {
|
||||||
contentType, _, err = readContentType(obj.PayloadSize(), func(sz uint64) (io.Reader, error) {
|
contentType, _, err = readContentType(obj.PayloadSize(), func(sz uint64) (io.Reader, error) {
|
||||||
prmRange := PrmObjectRange{
|
prmRange := frostfs.PrmObjectRange{
|
||||||
PrmAuth: PrmAuth{
|
PrmAuth: frostfs.PrmAuth{
|
||||||
BearerToken: btoken,
|
BearerToken: btoken,
|
||||||
},
|
},
|
||||||
Address: objectAddress,
|
Address: objectAddress,
|
||||||
|
|
|
@ -1,13 +1,30 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/multipart"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/multipart"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"github.com/minio/sio"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
FrostFSSystemMetadataPrefix = "S3-"
|
||||||
|
AttributeEncryptionAlgorithm = FrostFSSystemMetadataPrefix + "Algorithm"
|
||||||
|
AttributeHMACSalt = FrostFSSystemMetadataPrefix + "HMAC-Salt"
|
||||||
|
AttributeHMACKey = FrostFSSystemMetadataPrefix + "HMAC-Key"
|
||||||
|
AttributeMultipartObjectSize = FrostFSSystemMetadataPrefix + "Multipart-Object-Size"
|
||||||
|
)
|
||||||
|
|
||||||
// MultipartFile provides standard ReadCloser interface and also allows one to
|
// MultipartFile provides standard ReadCloser interface and also allows one to
|
||||||
// get file name, it's used for multipart uploads.
|
// get file name, it's used for multipart uploads.
|
||||||
type MultipartFile interface {
|
type MultipartFile interface {
|
||||||
|
@ -45,3 +62,145 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF
|
||||||
return part, nil
|
return part, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type getParams struct {
|
||||||
|
// payload range
|
||||||
|
off, ln uint64
|
||||||
|
|
||||||
|
objInfo *data.ObjectInfo
|
||||||
|
bktInfo *data.BucketInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
// getPayload returns initial payload if object is not multipart else composes new reader with parts data.
|
||||||
|
func (h *Handler) getPayload(p getPayloadParams) (io.ReadCloser, uint64, error) {
|
||||||
|
sizeValue, ok := p.attrs[AttributeMultipartObjectSize]
|
||||||
|
if !ok {
|
||||||
|
return p.obj.Payload, p.obj.Header.PayloadSize(), nil
|
||||||
|
}
|
||||||
|
cid, ok := p.obj.Header.ContainerID()
|
||||||
|
if !ok {
|
||||||
|
return nil, 0, errors.New("no container id set")
|
||||||
|
}
|
||||||
|
oid, ok := p.obj.Header.ID()
|
||||||
|
if !ok {
|
||||||
|
return nil, 0, errors.New("no object id set")
|
||||||
|
}
|
||||||
|
size, err := strconv.ParseUint(sizeValue, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
ctx := p.req.RequestCtx
|
||||||
|
params := getParams{
|
||||||
|
off: 0,
|
||||||
|
ln: 0,
|
||||||
|
objInfo: &data.ObjectInfo{
|
||||||
|
ID: oid,
|
||||||
|
CID: cid,
|
||||||
|
Bucket: p.bktinfo.Name,
|
||||||
|
Name: p.attrs["FilePath"],
|
||||||
|
Size: size,
|
||||||
|
Headers: p.attrs,
|
||||||
|
},
|
||||||
|
bktInfo: p.bktinfo,
|
||||||
|
}
|
||||||
|
payload, err := h.initMultipartReader(ctx, params)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return io.NopCloser(payload), size, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) initMultipartReader(ctx context.Context, p getParams) (io.Reader, error) {
|
||||||
|
combinedObj, err := h.frostfs.GetObject(ctx, frostfs.PrmObjectGet{
|
||||||
|
PrmAuth: frostfs.PrmAuth{BearerToken: bearerToken(ctx)},
|
||||||
|
Address: p.objInfo.Address(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("get combined object '%s': %w", p.objInfo.ID.EncodeToString(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var parts []*data.PartInfo
|
||||||
|
if err = json.NewDecoder(combinedObj.Payload).Decode(&parts); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal combined object parts: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
isEncrypted := FormEncryptionInfo(p.objInfo.Headers).Enabled
|
||||||
|
objParts := make([]frostfs.PartObj, len(parts))
|
||||||
|
for i, part := range parts {
|
||||||
|
size := part.Size
|
||||||
|
if isEncrypted {
|
||||||
|
if size, err = sio.EncryptedSize(part.Size); err != nil {
|
||||||
|
return nil, fmt.Errorf("compute encrypted size: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
objParts[i] = frostfs.PartObj{
|
||||||
|
OID: part.OID,
|
||||||
|
Size: size,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return frostfs.NewMultiObjectReader(ctx, frostfs.MultiObjectReaderConfig{
|
||||||
|
Handler: h,
|
||||||
|
Off: p.off,
|
||||||
|
Ln: p.ln,
|
||||||
|
Parts: objParts,
|
||||||
|
BktInfo: p.bktInfo,
|
||||||
|
Log: h.log,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitFrostFSObjectPayloadReader initializes payload reader of the FrostFS object.
|
||||||
|
// Zero range corresponds to full payload (panics if only offset is set).
|
||||||
|
func (h *Handler) InitFrostFSObjectPayloadReader(ctx context.Context, p frostfs.GetFrostFSParams) (io.Reader, error) {
|
||||||
|
var prmAuth frostfs.PrmAuth
|
||||||
|
|
||||||
|
var addr oid.Address
|
||||||
|
addr.SetContainer(p.BktInfo.CID)
|
||||||
|
addr.SetObject(p.Oid)
|
||||||
|
|
||||||
|
if p.Off+p.Ln != 0 {
|
||||||
|
prm := frostfs.PrmObjectRange{
|
||||||
|
PrmAuth: prmAuth,
|
||||||
|
Container: p.BktInfo.CID,
|
||||||
|
Object: p.Oid,
|
||||||
|
PayloadRange: [2]uint64{p.Off, p.Ln},
|
||||||
|
Address: addr,
|
||||||
|
}
|
||||||
|
|
||||||
|
return h.frostfs.RangeObject(ctx, prm)
|
||||||
|
}
|
||||||
|
|
||||||
|
prm := frostfs.PrmObjectGet{
|
||||||
|
PrmAuth: prmAuth,
|
||||||
|
Container: p.BktInfo.CID,
|
||||||
|
Object: p.Oid,
|
||||||
|
Address: addr,
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := h.frostfs.GetObject(ctx, prm)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.Payload, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObjectEncryption stores parsed object encryption headers.
|
||||||
|
type ObjectEncryption struct {
|
||||||
|
Enabled bool
|
||||||
|
Algorithm string
|
||||||
|
HMACKey string
|
||||||
|
HMACSalt string
|
||||||
|
}
|
||||||
|
|
||||||
|
func FormEncryptionInfo(headers map[string]string) ObjectEncryption {
|
||||||
|
algorithm := headers[AttributeEncryptionAlgorithm]
|
||||||
|
return ObjectEncryption{
|
||||||
|
Enabled: len(algorithm) > 0,
|
||||||
|
Algorithm: algorithm,
|
||||||
|
HMACKey: headers[AttributeHMACKey],
|
||||||
|
HMACSalt: headers[AttributeHMACSalt],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -5,11 +5,12 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"path"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
|
@ -47,19 +48,24 @@ func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (str
|
||||||
return http.DetectContentType(buf), buf, err // to not lose io.EOF
|
return http.DetectContentType(buf), buf, err // to not lose io.EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oid.Address) {
|
type getPayloadParams struct {
|
||||||
|
obj *frostfs.Object
|
||||||
|
req request
|
||||||
|
bktinfo *data.BucketInfo
|
||||||
|
attrs map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.Address, bktInfo *data.BucketInfo) {
|
||||||
var (
|
var (
|
||||||
err error
|
shouldDownload = req.QueryArgs().GetBool("download")
|
||||||
dis = "inline"
|
|
||||||
start = time.Now()
|
start = time.Now()
|
||||||
filename string
|
|
||||||
)
|
)
|
||||||
|
|
||||||
prm := PrmObjectGet{
|
prm := frostfs.PrmObjectGet{
|
||||||
PrmAuth: PrmAuth{
|
PrmAuth: frostfs.PrmAuth{
|
||||||
BearerToken: bearerToken(ctx),
|
BearerToken: bearerToken(ctx),
|
||||||
},
|
},
|
||||||
Address: objectAddress,
|
Address: objAddress,
|
||||||
}
|
}
|
||||||
|
|
||||||
rObj, err := h.frostfs.GetObject(ctx, prm)
|
rObj, err := h.frostfs.GetObject(ctx, prm)
|
||||||
|
@ -70,51 +76,40 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi
|
||||||
|
|
||||||
// we can't close reader in this function, so how to do it?
|
// we can't close reader in this function, so how to do it?
|
||||||
|
|
||||||
if req.Request.URI().QueryArgs().GetBool("download") {
|
attrs := makeAttributesMap(rObj.Header.Attributes())
|
||||||
dis = "attachment"
|
req.setAttributes(attrs)
|
||||||
}
|
|
||||||
|
|
||||||
payloadSize := rObj.Header.PayloadSize()
|
req.setIDs(rObj.Header)
|
||||||
|
req.setDisposition(shouldDownload, attrs[object.AttributeFileName])
|
||||||
|
|
||||||
req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10))
|
if err = req.setTimestamp(attrs[object.AttributeTimestamp]); err != nil {
|
||||||
var contentType string
|
req.log.Error(logs.CouldntParseCreationDate,
|
||||||
for _, attr := range rObj.Header.Attributes() {
|
zap.String("val", attrs[object.AttributeTimestamp]),
|
||||||
key := attr.Key()
|
|
||||||
val := attr.Value()
|
|
||||||
if !isValidToken(key) || !isValidValue(val) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
key = utils.BackwardTransformIfSystem(key)
|
|
||||||
|
|
||||||
req.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
|
|
||||||
switch key {
|
|
||||||
case object.AttributeFileName:
|
|
||||||
filename = val
|
|
||||||
case object.AttributeTimestamp:
|
|
||||||
value, err := strconv.ParseInt(val, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
req.log.Info(logs.CouldntParseCreationDate,
|
|
||||||
zap.String("key", key),
|
|
||||||
zap.String("val", val),
|
|
||||||
zap.Error(err))
|
zap.Error(err))
|
||||||
continue
|
response.Error(req.RequestCtx, "failed to convert timestamp: "+err.Error(), fasthttp.StatusInternalServerError)
|
||||||
}
|
|
||||||
req.Response.Header.Set(fasthttp.HeaderLastModified,
|
|
||||||
time.Unix(value, 0).UTC().Format(http.TimeFormat))
|
|
||||||
case object.AttributeContentType:
|
|
||||||
contentType = val
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
idsToResponse(&req.Response, &rObj.Header)
|
payloadParams := getPayloadParams{
|
||||||
|
obj: rObj,
|
||||||
|
req: req,
|
||||||
|
bktinfo: bktInfo,
|
||||||
|
attrs: attrs,
|
||||||
|
}
|
||||||
|
|
||||||
|
payload, payloadSize, err := h.getPayload(payloadParams)
|
||||||
|
if err != nil {
|
||||||
|
req.handleFrostFSErr(err, start)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10))
|
||||||
|
|
||||||
|
contentType := attrs[object.AttributeContentType]
|
||||||
if len(contentType) == 0 {
|
if len(contentType) == 0 {
|
||||||
// determine the Content-Type from the payload head
|
// determine the Content-Type from the payload head
|
||||||
var payloadHead []byte
|
var payloadHead []byte
|
||||||
|
|
||||||
contentType, payloadHead, err = readContentType(payloadSize, func(uint64) (io.Reader, error) {
|
contentType, payloadHead, err = readContentType(payloadSize, func(uint64) (io.Reader, error) {
|
||||||
return rObj.Payload, nil
|
return payload, nil
|
||||||
})
|
})
|
||||||
if err != nil && err != io.EOF {
|
if err != nil && err != io.EOF {
|
||||||
req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err))
|
req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err))
|
||||||
|
@ -126,16 +121,27 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi
|
||||||
var headReader io.Reader = bytes.NewReader(payloadHead)
|
var headReader io.Reader = bytes.NewReader(payloadHead)
|
||||||
|
|
||||||
if err != io.EOF { // otherwise, we've already read full payload
|
if err != io.EOF { // otherwise, we've already read full payload
|
||||||
headReader = io.MultiReader(headReader, rObj.Payload)
|
headReader = io.MultiReader(headReader, payload)
|
||||||
}
|
}
|
||||||
|
|
||||||
// note: we could do with io.Reader, but SetBodyStream below closes body stream
|
// note: we could do with io.Reader, but SetBodyStream below closes body stream
|
||||||
// if it implements io.Closer and that's useful for us.
|
// if it implements io.Closer and that's useful for us.
|
||||||
rObj.Payload = readCloser{headReader, rObj.Payload}
|
payload = readCloser{headReader, payload}
|
||||||
}
|
}
|
||||||
req.SetContentType(contentType)
|
req.SetContentType(contentType)
|
||||||
|
|
||||||
req.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename))
|
req.Response.SetBodyStream(payload, int(payloadSize))
|
||||||
|
}
|
||||||
req.Response.SetBodyStream(rObj.Payload, int(payloadSize))
|
|
||||||
|
func makeAttributesMap(attrs []object.Attribute) map[string]string {
|
||||||
|
attributes := make(map[string]string)
|
||||||
|
for _, attr := range attrs {
|
||||||
|
if !isValidToken(attr.Key()) || !isValidValue(attr.Value()) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := utils.BackwardTransformIfSystem(attr.Key())
|
||||||
|
|
||||||
|
attributes[key] = attr.Value()
|
||||||
|
}
|
||||||
|
return attributes
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
|
@ -131,8 +132,8 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) {
|
||||||
obj.SetOwnerID(*h.ownerID)
|
obj.SetOwnerID(*h.ownerID)
|
||||||
obj.SetAttributes(attributes...)
|
obj.SetAttributes(attributes...)
|
||||||
|
|
||||||
prm := PrmObjectCreate{
|
prm := frostfs.PrmObjectCreate{
|
||||||
PrmAuth: PrmAuth{
|
PrmAuth: frostfs.PrmAuth{
|
||||||
BearerToken: h.fetchBearerToken(ctx),
|
BearerToken: h.fetchBearerToken(ctx),
|
||||||
},
|
},
|
||||||
Object: obj,
|
Object: obj,
|
||||||
|
|
|
@ -2,14 +2,19 @@ package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"path"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -31,6 +36,43 @@ func (r *request) handleFrostFSErr(err error, start time.Time) {
|
||||||
response.Error(r.RequestCtx, msg, statusCode)
|
response.Error(r.RequestCtx, msg, statusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *request) setAttributes(attrs map[string]string) {
|
||||||
|
for key, val := range attrs {
|
||||||
|
r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *request) setIDs(obj object.Object) {
|
||||||
|
objID, _ := obj.ID()
|
||||||
|
cnrID, _ := obj.ContainerID()
|
||||||
|
r.Response.Header.Set(hdrObjectID, objID.String())
|
||||||
|
r.Response.Header.Set(hdrOwnerID, obj.OwnerID().String())
|
||||||
|
r.Response.Header.Set(hdrContainerID, cnrID.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *request) setDisposition(shouldDownload bool, filename string) {
|
||||||
|
const (
|
||||||
|
inlineDisposition = "inline"
|
||||||
|
attachmentDisposition = "attachment"
|
||||||
|
)
|
||||||
|
dis := inlineDisposition
|
||||||
|
if shouldDownload {
|
||||||
|
dis = attachmentDisposition
|
||||||
|
}
|
||||||
|
r.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *request) setTimestamp(timestamp string) error {
|
||||||
|
value, err := strconv.ParseInt(timestamp, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
r.Response.Header.Set(fasthttp.HeaderLastModified,
|
||||||
|
time.Unix(value, 0).UTC().Format(http.TimeFormat))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func bearerToken(ctx context.Context) *bearer.Token {
|
func bearerToken(ctx context.Context) *bearer.Token {
|
||||||
if tkn, err := tokens.LoadBearerToken(ctx); err == nil {
|
if tkn, err := tokens.LoadBearerToken(ctx); err == nil {
|
||||||
return tkn
|
return tkn
|
||||||
|
|
|
@ -7,15 +7,115 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
|
||||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"google.golang.org/grpc/codes"
|
"google.golang.org/grpc/codes"
|
||||||
"google.golang.org/grpc/status"
|
"google.golang.org/grpc/status"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ResObjectSearch interface {
|
||||||
|
Read(buf []oid.ID) (int, error)
|
||||||
|
Iterate(f func(oid.ID) bool) error
|
||||||
|
Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Object represents FrostFS object.
|
||||||
|
type Object struct {
|
||||||
|
// Object header (doesn't contain payload).
|
||||||
|
Header object.Object
|
||||||
|
|
||||||
|
// Object payload part encapsulated in io.Reader primitive.
|
||||||
|
// Returns ErrAccessDenied on read access violation.
|
||||||
|
Payload io.ReadCloser
|
||||||
|
}
|
||||||
|
|
||||||
|
type (
|
||||||
|
// PrmContainer groups parameters of FrostFS.Container operation.
|
||||||
|
PrmContainer struct {
|
||||||
|
// Container identifier.
|
||||||
|
ContainerID cid.ID
|
||||||
|
}
|
||||||
|
// PrmAuth groups authentication parameters for the FrostFS operation.
|
||||||
|
PrmAuth struct {
|
||||||
|
// Bearer token to be used for the operation. Overlaps PrivateKey. Optional.
|
||||||
|
BearerToken *bearer.Token
|
||||||
|
}
|
||||||
|
// PrmObjectHead groups parameters of FrostFS.HeadObject operation.
|
||||||
|
PrmObjectHead struct {
|
||||||
|
// Authentication parameters.
|
||||||
|
PrmAuth
|
||||||
|
|
||||||
|
// Address to read the object header from.
|
||||||
|
Address oid.Address
|
||||||
|
}
|
||||||
|
// PrmObjectGet groups parameters of FrostFS.GetObject operation.
|
||||||
|
PrmObjectGet struct {
|
||||||
|
// Authentication parameters.
|
||||||
|
PrmAuth
|
||||||
|
|
||||||
|
// Address to read the object header from.
|
||||||
|
Address oid.Address
|
||||||
|
Container cid.ID
|
||||||
|
Object oid.ID
|
||||||
|
}
|
||||||
|
// PrmObjectRange groups parameters of FrostFS.RangeObject operation.
|
||||||
|
PrmObjectRange struct {
|
||||||
|
// Authentication parameters.
|
||||||
|
PrmAuth
|
||||||
|
|
||||||
|
// Address to read the object header from.
|
||||||
|
Address oid.Address
|
||||||
|
|
||||||
|
// Offset-length range of the object payload to be read.
|
||||||
|
PayloadRange [2]uint64
|
||||||
|
|
||||||
|
Container cid.ID
|
||||||
|
|
||||||
|
Object oid.ID
|
||||||
|
}
|
||||||
|
// PrmObjectCreate groups parameters of FrostFS.CreateObject operation.
|
||||||
|
PrmObjectCreate struct {
|
||||||
|
// Authentication parameters.
|
||||||
|
PrmAuth
|
||||||
|
|
||||||
|
Object *object.Object
|
||||||
|
|
||||||
|
// Object payload encapsulated in io.Reader primitive.
|
||||||
|
Payload io.Reader
|
||||||
|
|
||||||
|
// Enables client side object preparing.
|
||||||
|
ClientCut bool
|
||||||
|
|
||||||
|
// Disables using Tillich-Zémor hash for payload.
|
||||||
|
WithoutHomomorphicHash bool
|
||||||
|
|
||||||
|
// Sets max buffer size to read payload.
|
||||||
|
BufferMaxSize uint64
|
||||||
|
}
|
||||||
|
// PrmObjectSearch groups parameters of FrostFS.sear SearchObjects operation.
|
||||||
|
PrmObjectSearch struct {
|
||||||
|
// Authentication parameters.
|
||||||
|
PrmAuth
|
||||||
|
|
||||||
|
// Container to select the objects from.
|
||||||
|
Container cid.ID
|
||||||
|
|
||||||
|
Filters object.SearchFilters
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrAccessDenied is returned from FrostFS in case of access violation.
|
||||||
|
ErrAccessDenied = errors.New("access denied")
|
||||||
|
// ErrGatewayTimeout is returned from FrostFS in case of timeout, deadline exceeded etc.
|
||||||
|
ErrGatewayTimeout = errors.New("gateway timeout")
|
||||||
)
|
)
|
||||||
|
|
||||||
// FrostFS represents virtual connection to the FrostFS network.
|
// FrostFS represents virtual connection to the FrostFS network.
|
||||||
|
@ -33,9 +133,9 @@ func NewFrostFS(p *pool.Pool) *FrostFS {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Container implements frostfs.FrostFS interface method.
|
// Container implements frostfs.FrostFS interface method.
|
||||||
func (x *FrostFS) Container(ctx context.Context, layerPrm handler.PrmContainer) (*container.Container, error) {
|
func (x *FrostFS) Container(ctx context.Context, containerPrm PrmContainer) (*container.Container, error) {
|
||||||
prm := pool.PrmContainerGet{
|
prm := pool.PrmContainerGet{
|
||||||
ContainerID: layerPrm.ContainerID,
|
ContainerID: containerPrm.ContainerID,
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := x.pool.GetContainer(ctx, prm)
|
res, err := x.pool.GetContainer(ctx, prm)
|
||||||
|
@ -47,7 +147,7 @@ func (x *FrostFS) Container(ctx context.Context, layerPrm handler.PrmContainer)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateObject implements frostfs.FrostFS interface method.
|
// CreateObject implements frostfs.FrostFS interface method.
|
||||||
func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) (oid.ID, error) {
|
func (x *FrostFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.ID, error) {
|
||||||
var prmPut pool.PrmObjectPut
|
var prmPut pool.PrmObjectPut
|
||||||
prmPut.SetHeader(*prm.Object)
|
prmPut.SetHeader(*prm.Object)
|
||||||
prmPut.SetPayload(prm.Payload)
|
prmPut.SetPayload(prm.Payload)
|
||||||
|
@ -78,7 +178,7 @@ func (x payloadReader) Read(p []byte) (int, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// HeadObject implements frostfs.FrostFS interface method.
|
// HeadObject implements frostfs.FrostFS interface method.
|
||||||
func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*object.Object, error) {
|
func (x *FrostFS) HeadObject(ctx context.Context, prm PrmObjectHead) (*object.Object, error) {
|
||||||
var prmHead pool.PrmObjectHead
|
var prmHead pool.PrmObjectHead
|
||||||
prmHead.SetAddress(prm.Address)
|
prmHead.SetAddress(prm.Address)
|
||||||
|
|
||||||
|
@ -95,7 +195,7 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetObject implements frostfs.FrostFS interface method.
|
// GetObject implements frostfs.FrostFS interface method.
|
||||||
func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*handler.Object, error) {
|
func (x *FrostFS) GetObject(ctx context.Context, prm PrmObjectGet) (*Object, error) {
|
||||||
var prmGet pool.PrmObjectGet
|
var prmGet pool.PrmObjectGet
|
||||||
prmGet.SetAddress(prm.Address)
|
prmGet.SetAddress(prm.Address)
|
||||||
|
|
||||||
|
@ -108,14 +208,14 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han
|
||||||
return nil, handleObjectError("init full object reading via connection pool", err)
|
return nil, handleObjectError("init full object reading via connection pool", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &handler.Object{
|
return &Object{
|
||||||
Header: res.Header,
|
Header: res.Header,
|
||||||
Payload: res.Payload,
|
Payload: res.Payload,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RangeObject implements frostfs.FrostFS interface method.
|
// RangeObject implements frostfs.FrostFS interface method.
|
||||||
func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) (io.ReadCloser, error) {
|
func (x *FrostFS) RangeObject(ctx context.Context, prm PrmObjectRange) (io.ReadCloser, error) {
|
||||||
var prmRange pool.PrmObjectRange
|
var prmRange pool.PrmObjectRange
|
||||||
prmRange.SetAddress(prm.Address)
|
prmRange.SetAddress(prm.Address)
|
||||||
prmRange.SetOffset(prm.PayloadRange[0])
|
prmRange.SetOffset(prm.PayloadRange[0])
|
||||||
|
@ -134,7 +234,7 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) (
|
||||||
}
|
}
|
||||||
|
|
||||||
// SearchObjects implements frostfs.FrostFS interface method.
|
// SearchObjects implements frostfs.FrostFS interface method.
|
||||||
func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch) (handler.ResObjectSearch, error) {
|
func (x *FrostFS) SearchObjects(ctx context.Context, prm PrmObjectSearch) (ResObjectSearch, error) {
|
||||||
var prmSearch pool.PrmObjectSearch
|
var prmSearch pool.PrmObjectSearch
|
||||||
prmSearch.SetContainerID(prm.Container)
|
prmSearch.SetContainerID(prm.Container)
|
||||||
prmSearch.SetFilters(prm.Filters)
|
prmSearch.SetFilters(prm.Filters)
|
||||||
|
@ -202,11 +302,11 @@ func handleObjectError(msg string, err error) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
if reason, ok := IsErrObjectAccessDenied(err); ok {
|
if reason, ok := IsErrObjectAccessDenied(err); ok {
|
||||||
return fmt.Errorf("%s: %w: %s", msg, handler.ErrAccessDenied, reason)
|
return fmt.Errorf("%s: %w: %s", msg, ErrAccessDenied, reason)
|
||||||
}
|
}
|
||||||
|
|
||||||
if IsTimeoutError(err) {
|
if IsTimeoutError(err) {
|
||||||
return fmt.Errorf("%s: %w: %s", msg, handler.ErrGatewayTimeout, err.Error())
|
return fmt.Errorf("%s: %w: %s", msg, ErrGatewayTimeout, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf("%s: %w", msg, err)
|
return fmt.Errorf("%s: %w", msg, err)
|
168
internal/service/frostfs/multi_object_reader.go
Normal file
168
internal/service/frostfs/multi_object_reader.go
Normal file
|
@ -0,0 +1,168 @@
|
||||||
|
package frostfs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type GetFrostFSParams struct {
|
||||||
|
// payload range
|
||||||
|
Off, Ln uint64
|
||||||
|
|
||||||
|
Oid oid.ID
|
||||||
|
BktInfo *data.BucketInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
type PartObj struct {
|
||||||
|
OID oid.ID
|
||||||
|
Size uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
type readerInitiator interface {
|
||||||
|
InitFrostFSObjectPayloadReader(ctx context.Context, p GetFrostFSParams) (io.Reader, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MultiObjectReader implements io.Reader of payloads of the object list stored in the FrostFS network.
|
||||||
|
type MultiObjectReader struct {
|
||||||
|
ctx context.Context
|
||||||
|
log *zap.Logger
|
||||||
|
|
||||||
|
layer readerInitiator
|
||||||
|
|
||||||
|
startPartOffset uint64
|
||||||
|
endPartLength uint64
|
||||||
|
|
||||||
|
prm GetFrostFSParams
|
||||||
|
|
||||||
|
curIndex int
|
||||||
|
curReader io.Reader
|
||||||
|
|
||||||
|
parts []PartObj
|
||||||
|
}
|
||||||
|
|
||||||
|
type MultiObjectReaderConfig struct {
|
||||||
|
Handler readerInitiator
|
||||||
|
Log *zap.Logger
|
||||||
|
|
||||||
|
// the offset of complete object and total size to read
|
||||||
|
Off, Ln uint64
|
||||||
|
|
||||||
|
BktInfo *data.BucketInfo
|
||||||
|
Parts []PartObj
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
errOffsetIsOutOfRange = errors.New("offset is out of payload range")
|
||||||
|
errLengthIsOutOfRange = errors.New("length is out of payload range")
|
||||||
|
errEmptyPartsList = errors.New("empty parts list")
|
||||||
|
errorZeroRangeLength = errors.New("zero range length")
|
||||||
|
)
|
||||||
|
|
||||||
|
func NewMultiObjectReader(ctx context.Context, cfg MultiObjectReaderConfig) (*MultiObjectReader, error) {
|
||||||
|
if len(cfg.Parts) == 0 {
|
||||||
|
return nil, errEmptyPartsList
|
||||||
|
}
|
||||||
|
|
||||||
|
r := &MultiObjectReader{
|
||||||
|
ctx: ctx,
|
||||||
|
layer: cfg.Handler,
|
||||||
|
prm: GetFrostFSParams{
|
||||||
|
BktInfo: cfg.BktInfo,
|
||||||
|
},
|
||||||
|
parts: cfg.Parts,
|
||||||
|
log: cfg.Log,
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Off+cfg.Ln == 0 {
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Off > 0 && cfg.Ln == 0 {
|
||||||
|
return nil, errorZeroRangeLength
|
||||||
|
}
|
||||||
|
|
||||||
|
startPartIndex, startPartOffset := findStartPart(cfg)
|
||||||
|
if startPartIndex == -1 {
|
||||||
|
return nil, errOffsetIsOutOfRange
|
||||||
|
}
|
||||||
|
r.startPartOffset = startPartOffset
|
||||||
|
|
||||||
|
endPartIndex, endPartLength := findEndPart(cfg)
|
||||||
|
if endPartIndex == -1 {
|
||||||
|
return nil, errLengthIsOutOfRange
|
||||||
|
}
|
||||||
|
r.endPartLength = endPartLength
|
||||||
|
|
||||||
|
r.parts = cfg.Parts[startPartIndex : endPartIndex+1]
|
||||||
|
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func findStartPart(cfg MultiObjectReaderConfig) (index int, offset uint64) {
|
||||||
|
position := cfg.Off
|
||||||
|
for i, part := range cfg.Parts {
|
||||||
|
// Strict inequality when searching for start position to avoid reading zero length part.
|
||||||
|
if position < part.Size {
|
||||||
|
return i, position
|
||||||
|
}
|
||||||
|
position -= part.Size
|
||||||
|
}
|
||||||
|
|
||||||
|
return -1, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func findEndPart(cfg MultiObjectReaderConfig) (index int, length uint64) {
|
||||||
|
position := cfg.Off + cfg.Ln
|
||||||
|
for i, part := range cfg.Parts {
|
||||||
|
// Non-strict inequality when searching for end position to avoid out of payload range error.
|
||||||
|
if position <= part.Size {
|
||||||
|
return i, position
|
||||||
|
}
|
||||||
|
position -= part.Size
|
||||||
|
}
|
||||||
|
|
||||||
|
return -1, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *MultiObjectReader) Read(p []byte) (n int, err error) {
|
||||||
|
if x.curReader != nil {
|
||||||
|
n, err = x.curReader.Read(p)
|
||||||
|
if !errors.Is(err, io.EOF) {
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
x.curIndex++
|
||||||
|
}
|
||||||
|
|
||||||
|
if x.curIndex == len(x.parts) {
|
||||||
|
return n, io.EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
x.prm.Oid = x.parts[x.curIndex].OID
|
||||||
|
|
||||||
|
if x.curIndex == 0 {
|
||||||
|
x.prm.Off = x.startPartOffset
|
||||||
|
x.prm.Ln = x.parts[x.curIndex].Size - x.startPartOffset
|
||||||
|
}
|
||||||
|
|
||||||
|
if x.curIndex == len(x.parts)-1 {
|
||||||
|
x.prm.Ln = x.endPartLength - x.prm.Off
|
||||||
|
}
|
||||||
|
|
||||||
|
x.curReader, err = x.layer.InitFrostFSObjectPayloadReader(x.ctx, x.prm)
|
||||||
|
if err != nil {
|
||||||
|
return n, fmt.Errorf("init payload reader for the next part: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
x.prm.Off = 0
|
||||||
|
x.prm.Ln = 0
|
||||||
|
|
||||||
|
next, err := x.Read(p[n:])
|
||||||
|
|
||||||
|
return n + next, err
|
||||||
|
}
|
137
internal/service/frostfs/multi_object_reader_test.go
Normal file
137
internal/service/frostfs/multi_object_reader_test.go
Normal file
|
@ -0,0 +1,137 @@
|
||||||
|
package frostfs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
type readerInitiatorMock struct {
|
||||||
|
parts map[oid.ID][]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *readerInitiatorMock) InitFrostFSObjectPayloadReader(_ context.Context, p GetFrostFSParams) (io.Reader, error) {
|
||||||
|
partPayload, ok := r.parts[p.Oid]
|
||||||
|
if !ok {
|
||||||
|
return nil, errors.New("part not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.Off+p.Ln == 0 {
|
||||||
|
return bytes.NewReader(partPayload), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.Off > uint64(len(partPayload)-1) {
|
||||||
|
return nil, fmt.Errorf("invalid offset: %d/%d", p.Off, len(partPayload))
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.Off+p.Ln > uint64(len(partPayload)) {
|
||||||
|
return nil, fmt.Errorf("invalid range: %d-%d/%d", p.Off, p.Off+p.Ln, len(partPayload))
|
||||||
|
}
|
||||||
|
|
||||||
|
return bytes.NewReader(partPayload[p.Off : p.Off+p.Ln]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareDataReader() ([]byte, []PartObj, *readerInitiatorMock) {
|
||||||
|
mockInitReader := &readerInitiatorMock{
|
||||||
|
parts: map[oid.ID][]byte{
|
||||||
|
oidtest.ID(): []byte("first part 1"),
|
||||||
|
oidtest.ID(): []byte("second part 2"),
|
||||||
|
oidtest.ID(): []byte("third part 3"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var fullPayload []byte
|
||||||
|
parts := make([]PartObj, 0, len(mockInitReader.parts))
|
||||||
|
for id, payload := range mockInitReader.parts {
|
||||||
|
parts = append(parts, PartObj{OID: id, Size: uint64(len(payload))})
|
||||||
|
fullPayload = append(fullPayload, payload...)
|
||||||
|
}
|
||||||
|
|
||||||
|
return fullPayload, parts, mockInitReader
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultiReader(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
fullPayload, parts, mockInitReader := prepareDataReader()
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
off uint64
|
||||||
|
ln uint64
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "simple read all",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "simple read with length",
|
||||||
|
ln: uint64(len(fullPayload)),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "middle of parts",
|
||||||
|
off: parts[0].Size + 2,
|
||||||
|
ln: 4,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "first and second",
|
||||||
|
off: parts[0].Size - 4,
|
||||||
|
ln: 8,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "first and third",
|
||||||
|
off: parts[0].Size - 4,
|
||||||
|
ln: parts[1].Size + 8,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "second part",
|
||||||
|
off: parts[0].Size,
|
||||||
|
ln: parts[1].Size,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "second and third",
|
||||||
|
off: parts[0].Size,
|
||||||
|
ln: parts[1].Size + parts[2].Size,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "offset out of range",
|
||||||
|
off: uint64(len(fullPayload) + 1),
|
||||||
|
ln: 1,
|
||||||
|
err: errOffsetIsOutOfRange,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "zero length",
|
||||||
|
off: parts[1].Size + 1,
|
||||||
|
ln: 0,
|
||||||
|
err: errorZeroRangeLength,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
multiReader, err := NewMultiObjectReader(ctx, MultiObjectReaderConfig{
|
||||||
|
Handler: mockInitReader,
|
||||||
|
Parts: parts,
|
||||||
|
Off: tc.off,
|
||||||
|
Ln: tc.ln,
|
||||||
|
})
|
||||||
|
require.ErrorIs(t, err, tc.err)
|
||||||
|
|
||||||
|
if tc.err == nil {
|
||||||
|
off := tc.off
|
||||||
|
ln := tc.ln
|
||||||
|
if off+ln == 0 {
|
||||||
|
ln = uint64(len(fullPayload))
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(multiReader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, fullPayload[off:off+ln], data)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,4 +1,4 @@
|
||||||
package services
|
package frostfs
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
Loading…
Reference in a new issue