diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 561598f..15d6305 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -17,11 +17,10 @@ import ( v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" @@ -400,7 +399,7 @@ func (a *app) setHealthStatus() { } func (a *app) Serve() { - handler := handler.New(a.AppParams(), a.settings, tree.NewTree(services.NewPoolWrapper(a.treePool))) + handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool))) // Configure router. a.configureRouter(handler) diff --git a/go.mod b/go.mod index accedfb..606be17 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,8 @@ require ( git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 github.com/fasthttp/router v1.4.1 - github.com/nspcc-dev/neo-go v0.106.2 + github.com/minio/sio v0.4.1 + github.com/nspcc-dev/neo-go v0.106.3 github.com/prometheus/client_golang v1.19.0 github.com/prometheus/client_model v0.5.0 github.com/spf13/pflag v1.0.5 @@ -28,7 +29,7 @@ require ( ) require ( - git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e // indirect + git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.4 // indirect git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect @@ -43,7 +44,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/containerd v1.6.2 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/distribution v2.8.1+incompatible // indirect @@ -56,13 +57,14 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/snappy v0.0.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/holiman/uint256 v1.2.4 // indirect github.com/klauspost/compress v1.16.4 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -72,12 +74,13 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240727093519-1a48f1ce43ec // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.1.1 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.48.0 // indirect @@ -89,10 +92,11 @@ require ( github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect github.com/twmb/murmur3 v1.1.8 // indirect - github.com/urfave/cli v1.22.5 // indirect + github.com/urfave/cli/v2 v2.27.2 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect go.etcd.io/bbolt v1.3.9 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect @@ -103,11 +107,11 @@ require ( go.opentelemetry.io/otel/sdk v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.23.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect diff --git a/go.sum b/go.sum index 07eccfa..424b887 100644 --- a/go.sum +++ b/go.sum @@ -39,8 +39,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 h1:XxvwQKJT/f16qS3df5PBQPRYKkhy0/A7zH6644QpKD0= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.4 h1:HYukyt9WGKBnkexnIlzg6Am3SvH48QZbtIxt0YeObBs= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.4/go.mod h1:Y2Xorxc8SBO4phoek7n3XxaPZz5rIrFgDsU4TOjmlGA= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= @@ -280,8 +280,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= @@ -438,9 +438,9 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -587,6 +587,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/minio/sio v0.4.1 h1:EMe3YBC1nf+sRQia65Rutxi+Z554XPV0dt8BIBA+a/0= +github.com/minio/sio v0.4.1/go.mod h1:oBSjJeGbBdRMZZwna07sX9EFzZy+ywu5aofRiV1g79I= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -624,12 +626,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nspcc-dev/dbft v0.2.0 h1:sDwsQES600OSIMncV176t2SX5OvB14lzeOAyKFOkbMI= +github.com/nspcc-dev/dbft v0.2.0/go.mod h1:oFE6paSC/yfFh9mcNU6MheMGOYXK9+sPiRk3YMoz49o= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc= -github.com/nspcc-dev/neo-go v0.106.2 h1:KXSJ2J5Oacc7LrX3r4jvnC8ihKqHs5NB21q4f2S3r9o= -github.com/nspcc-dev/neo-go v0.106.2/go.mod h1:Ojwfx3/lv0VTeEHMpQ17g0wTnXcCSoFQVq5GEeCZmGo= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d h1:Vcb7YkZuUSSIC+WF/xV3UDfHbAxZgyT2zGleJP3Ig5k= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d/go.mod h1:/vrbWSHc7YS1KSYhVOyyeucXW/e+1DkVBOgnBEXUCeY= +github.com/nspcc-dev/neo-go v0.106.3 h1:HEyhgkjQY+HfBzotMJ12xx2VuOUphkngZ4kEkjvXDtE= +github.com/nspcc-dev/neo-go v0.106.3/go.mod h1:3vEwJ2ld12N7HRGCaH/l/7EwopplC/+8XdIdPDNmD/M= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240727093519-1a48f1ce43ec h1:vDrbVXF2+2uP0RlkZmem3QYATcXCu9BzzGGCNsNcK7Q= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240727093519-1a48f1ce43ec/go.mod h1:/vrbWSHc7YS1KSYhVOyyeucXW/e+1DkVBOgnBEXUCeY= github.com/nspcc-dev/rfc6979 v0.2.1 h1:8wWxkamHWFmO790GsewSoKUSJjVnL1fmdRpokU/RgRM= github.com/nspcc-dev/rfc6979 v0.2.1/go.mod h1:Tk7h5kyUWkhjyO3zUgFFhy1v2vQv3BvQEntakdtqrWc= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= @@ -690,6 +694,8 @@ github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrap github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -813,8 +819,8 @@ github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNG github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= +github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBNGN0TYb/7oKIPVn15JA= github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk= @@ -829,8 +835,8 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= -github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= +github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA= @@ -850,6 +856,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -921,8 +929,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1127,12 +1135,12 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1142,8 +1150,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/internal/data/object.go b/internal/data/object.go new file mode 100644 index 0000000..2575218 --- /dev/null +++ b/internal/data/object.go @@ -0,0 +1,41 @@ +package data + +import ( + "time" + + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +type ( + ObjectInfo struct { + ID oid.ID + CID cid.ID + + Bucket string + Name string + Size uint64 + Headers map[string]string + } + + // PartInfo is upload information about part. + PartInfo struct { + Key string `json:"key"` + UploadID string `json:"uploadId"` + Number int `json:"number"` + OID oid.ID `json:"oid"` + Size uint64 `json:"size"` + ETag string `json:"etag"` + MD5 string `json:"md5"` + Created time.Time `json:"created"` + } +) + +// Address returns object address. +func (o *ObjectInfo) Address() oid.Address { + var addr oid.Address + addr.SetContainer(o.CID) + addr.SetObject(o.ID) + + return addr +} diff --git a/internal/handler/download.go b/internal/handler/download.go index 88109a6..29aeeac 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -11,6 +11,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" @@ -45,13 +46,13 @@ func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { h.byAttribute(c, h.receiveFile) } -func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) { +func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op object.SearchMatchType) (frostfs.ResObjectSearch, error) { filters := object.NewSearchFilters() filters.AddRootFilter() filters.AddFilter(key, val, op) - prm := PrmObjectSearch{ - PrmAuth: PrmAuth{ + prm := frostfs.PrmObjectSearch{ + PrmAuth: frostfs.PrmAuth{ BearerToken: bearerToken(ctx), }, Container: *cnrID, @@ -153,8 +154,8 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { } func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error { - prm := PrmObjectGet{ - PrmAuth: PrmAuth{ + prm := frostfs.PrmObjectGet{ + PrmAuth: frostfs.PrmAuth{ BearerToken: btoken, }, Address: addr, diff --git a/internal/handler/frostfs_mock.go b/internal/handler/frostfs_mock.go index 9f4378a..6d9686f 100644 --- a/internal/handler/frostfs_mock.go +++ b/internal/handler/frostfs_mock.go @@ -9,6 +9,7 @@ import ( "io" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" @@ -58,7 +59,7 @@ func (t *TestFrostFS) AllowUserOperation(cnrID cid.ID, userID user.ID, op acl.Op t.accessList[fmt.Sprintf("%s/%s/%s/%s", cnrID, userID, op, objID)] = true } -func (t *TestFrostFS) Container(_ context.Context, prm PrmContainer) (*container.Container, error) { +func (t *TestFrostFS) Container(_ context.Context, prm frostfs.PrmContainer) (*container.Container, error) { for k, v := range t.containers { if k == prm.ContainerID.EncodeToString() { return v, nil @@ -85,7 +86,7 @@ func (t *TestFrostFS) retrieveObject(addr oid.Address, btoken *bearer.Token) (*o owner := t.requestOwner(btoken) if !t.isAllowed(addr.Container(), owner, acl.OpObjectGet, addr.Object()) { - return nil, ErrAccessDenied + return nil, frostfs.ErrAccessDenied } return obj, nil @@ -94,23 +95,23 @@ func (t *TestFrostFS) retrieveObject(addr oid.Address, btoken *bearer.Token) (*o return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, addr) } -func (t *TestFrostFS) HeadObject(_ context.Context, prm PrmObjectHead) (*object.Object, error) { +func (t *TestFrostFS) HeadObject(_ context.Context, prm frostfs.PrmObjectHead) (*object.Object, error) { return t.retrieveObject(prm.Address, prm.BearerToken) } -func (t *TestFrostFS) GetObject(_ context.Context, prm PrmObjectGet) (*Object, error) { +func (t *TestFrostFS) GetObject(_ context.Context, prm frostfs.PrmObjectGet) (*frostfs.Object, error) { obj, err := t.retrieveObject(prm.Address, prm.BearerToken) if err != nil { return nil, err } - return &Object{ + return &frostfs.Object{ Header: *obj, Payload: io.NopCloser(bytes.NewReader(obj.Payload())), }, nil } -func (t *TestFrostFS) RangeObject(_ context.Context, prm PrmObjectRange) (io.ReadCloser, error) { +func (t *TestFrostFS) RangeObject(_ context.Context, prm frostfs.PrmObjectRange) (io.ReadCloser, error) { obj, err := t.retrieveObject(prm.Address, prm.BearerToken) if err != nil { return nil, err @@ -121,7 +122,7 @@ func (t *TestFrostFS) RangeObject(_ context.Context, prm PrmObjectRange) (io.Rea return io.NopCloser(bytes.NewReader(payload)), nil } -func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID, error) { +func (t *TestFrostFS) CreateObject(_ context.Context, prm frostfs.PrmObjectCreate) (oid.ID, error) { b := make([]byte, 32) if _, err := io.ReadFull(rand.Reader, b); err != nil { return oid.ID{}, err @@ -158,7 +159,7 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid. owner := t.requestOwner(prm.BearerToken) if !t.isAllowed(cnrID, owner, acl.OpObjectPut, objID) { - return oid.ID{}, ErrAccessDenied + return oid.ID{}, frostfs.ErrAccessDenied } addr := newAddress(cnrID, objID) @@ -195,9 +196,9 @@ func (r *resObjectSearchMock) Iterate(f func(oid.ID) bool) error { func (r *resObjectSearchMock) Close() {} -func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) (ResObjectSearch, error) { +func (t *TestFrostFS) SearchObjects(_ context.Context, prm frostfs.PrmObjectSearch) (frostfs.ResObjectSearch, error) { if !t.isAllowed(prm.Container, t.requestOwner(prm.BearerToken), acl.OpObjectSearch, oid.ID{}) { - return nil, ErrAccessDenied + return nil, frostfs.ErrAccessDenied } cidStr := prm.Container.EncodeToString() diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4de9d9a..fa7cc43 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -12,10 +12,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -34,110 +34,14 @@ type Config interface { NamespaceHeader() string } -// PrmContainer groups parameters of FrostFS.Container operation. -type PrmContainer struct { - // Container identifier. - ContainerID cid.ID -} - -// PrmAuth groups authentication parameters for the FrostFS operation. -type PrmAuth struct { - // Bearer token to be used for the operation. Overlaps PrivateKey. Optional. - BearerToken *bearer.Token -} - -// PrmObjectHead groups parameters of FrostFS.HeadObject operation. -type PrmObjectHead struct { - // Authentication parameters. - PrmAuth - - // Address to read the object header from. - Address oid.Address -} - -// PrmObjectGet groups parameters of FrostFS.GetObject operation. -type PrmObjectGet struct { - // Authentication parameters. - PrmAuth - - // Address to read the object header from. - Address oid.Address -} - -// PrmObjectRange groups parameters of FrostFS.RangeObject operation. -type PrmObjectRange struct { - // Authentication parameters. - PrmAuth - - // Address to read the object header from. - Address oid.Address - - // Offset-length range of the object payload to be read. - PayloadRange [2]uint64 -} - -// Object represents FrostFS object. -type Object struct { - // Object header (doesn't contain payload). - Header object.Object - - // Object payload part encapsulated in io.Reader primitive. - // Returns ErrAccessDenied on read access violation. - Payload io.ReadCloser -} - -// PrmObjectCreate groups parameters of FrostFS.CreateObject operation. -type PrmObjectCreate struct { - // Authentication parameters. - PrmAuth - - Object *object.Object - - // Object payload encapsulated in io.Reader primitive. - Payload io.Reader - - // Enables client side object preparing. - ClientCut bool - - // Disables using Tillich-Zémor hash for payload. - WithoutHomomorphicHash bool - - // Sets max buffer size to read payload. - BufferMaxSize uint64 -} - -// PrmObjectSearch groups parameters of FrostFS.sear SearchObjects operation. -type PrmObjectSearch struct { - // Authentication parameters. - PrmAuth - - // Container to select the objects from. - Container cid.ID - - Filters object.SearchFilters -} - -type ResObjectSearch interface { - Read(buf []oid.ID) (int, error) - Iterate(f func(oid.ID) bool) error - Close() -} - -var ( - // ErrAccessDenied is returned from FrostFS in case of access violation. - ErrAccessDenied = errors.New("access denied") - // ErrGatewayTimeout is returned from FrostFS in case of timeout, deadline exceeded etc. - ErrGatewayTimeout = errors.New("gateway timeout") -) - // FrostFS represents virtual connection to FrostFS network. type FrostFS interface { - Container(context.Context, PrmContainer) (*container.Container, error) - HeadObject(context.Context, PrmObjectHead) (*object.Object, error) - GetObject(context.Context, PrmObjectGet) (*Object, error) - RangeObject(context.Context, PrmObjectRange) (io.ReadCloser, error) - CreateObject(context.Context, PrmObjectCreate) (oid.ID, error) - SearchObjects(context.Context, PrmObjectSearch) (ResObjectSearch, error) + Container(context.Context, frostfs.PrmContainer) (*container.Container, error) + HeadObject(context.Context, frostfs.PrmObjectHead) (*object.Object, error) + GetObject(context.Context, frostfs.PrmObjectGet) (*frostfs.Object, error) + RangeObject(context.Context, frostfs.PrmObjectRange) (io.ReadCloser, error) + CreateObject(context.Context, frostfs.PrmObjectCreate) (oid.ID, error) + SearchObjects(context.Context, frostfs.PrmObjectSearch) (frostfs.ResObjectSearch, error) utils.EpochInfoFetcher } @@ -177,7 +81,7 @@ func New(params *AppParams, config Config, tree *tree.Tree) *Handler { // byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { +func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address, *data.BucketInfo)) { var ( idCnr, _ = c.UserValue("cid").(string) idObj, _ = c.UserValue("oid").(string) @@ -203,12 +107,12 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ addr.SetContainer(bktInfo.CID) addr.SetObject(*objID) - f(ctx, *h.newRequest(c, log), addr) + f(ctx, *h.newRequest(c, log), addr, bktInfo) } // byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { +func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address, *data.BucketInfo)) { var ( bucketname = req.UserValue("cid").(string) key = req.UserValue("oid").(string) @@ -250,11 +154,11 @@ func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, addr.SetContainer(bktInfo.CID) addr.SetObject(foundOid.OID) - f(ctx, *h.newRequest(req, log), addr) + f(ctx, *h.newRequest(req, log), addr, bktInfo) } // byAttribute is a wrapper similar to byAddress. -func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { +func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address, *data.BucketInfo)) { scid, _ := c.UserValue("cid").(string) key, _ := c.UserValue("attr_key").(string) val, _ := c.UserValue("attr_val").(string) @@ -311,7 +215,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re addrObj.SetContainer(bktInfo.CID) addrObj.SetObject(buf[0]) - f(ctx, *h.newRequest(c, log), addrObj) + f(ctx, *h.newRequest(c, log), addrObj, bktInfo) } // resolveContainer decode container id, if it's not a valid container id @@ -359,7 +263,7 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log * } func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) { - prm := PrmContainer{ContainerID: cnrID} + prm := frostfs.PrmContainer{ContainerID: cnrID} res, err := h.frostfs.Container(ctx, prm) if err != nil { return nil, fmt.Errorf("get frostfs container '%s': %w", cnrID.String(), err) diff --git a/internal/handler/head.go b/internal/handler/head.go index f0a1e94..98672ed 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -7,7 +7,9 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -24,13 +26,13 @@ const ( hdrContainerID = "X-Container-Id" ) -func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid.Address) { +func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid.Address, _ *data.BucketInfo) { var start = time.Now() btoken := bearerToken(ctx) - prm := PrmObjectHead{ - PrmAuth: PrmAuth{ + prm := frostfs.PrmObjectHead{ + PrmAuth: frostfs.PrmAuth{ BearerToken: btoken, }, Address: objectAddress, @@ -74,8 +76,8 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid if len(contentType) == 0 { contentType, _, err = readContentType(obj.PayloadSize(), func(sz uint64) (io.Reader, error) { - prmRange := PrmObjectRange{ - PrmAuth: PrmAuth{ + prmRange := frostfs.PrmObjectRange{ + PrmAuth: frostfs.PrmAuth{ BearerToken: btoken, }, Address: objectAddress, diff --git a/internal/handler/multipart.go b/internal/handler/multipart.go index de9242f..d745094 100644 --- a/internal/handler/multipart.go +++ b/internal/handler/multipart.go @@ -1,13 +1,29 @@ package handler import ( + "context" + "encoding/json" + "fmt" "io" + "strconv" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/multipart" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/minio/sio" "go.uber.org/zap" ) +const ( + FrostFSSystemMetadataPrefix = "S3-" + AttributeEncryptionAlgorithm = FrostFSSystemMetadataPrefix + "Algorithm" + AttributeHMACSalt = FrostFSSystemMetadataPrefix + "HMAC-Salt" + AttributeHMACKey = FrostFSSystemMetadataPrefix + "HMAC-Key" + AttributeMultipartObjectSize = FrostFSSystemMetadataPrefix + "Multipart-Object-Size" +) + // MultipartFile provides standard ReadCloser interface and also allows one to // get file name, it's used for multipart uploads. type MultipartFile interface { @@ -45,3 +61,139 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF return part, nil } } + +type getParams struct { + // payload range + off, ln uint64 + + objInfo *data.ObjectInfo + bktInfo *data.BucketInfo +} + +// getPayload returns initial payload if object is not multipart else composes new reader with parts data. +func (h *Handler) getPayload(p getPayloadParams) (io.ReadCloser, uint64, error) { + sizeValue, ok := p.attrs[AttributeMultipartObjectSize] + if !ok { + return p.obj.Payload, p.obj.Header.PayloadSize(), nil + } + cid, _ := p.obj.Header.ContainerID() + oid, _ := p.obj.Header.ID() + size, err := strconv.ParseUint(sizeValue, 10, 64) + if err != nil { + return nil, 0, err + } + ctx := p.req.RequestCtx + params := getParams{ + off: 0, + ln: 0, + objInfo: &data.ObjectInfo{ + ID: oid, + CID: cid, + Bucket: p.bktinfo.Name, + Name: p.attrs["FilePath"], + Size: size, + Headers: p.attrs, + }, + bktInfo: p.bktinfo, + } + payload, err := h.initMultipartReader(ctx, params) + if err != nil { + return nil, 0, err + } + + return io.NopCloser(payload), size, nil +} + +func (h *Handler) initMultipartReader(ctx context.Context, p getParams) (io.Reader, error) { + combinedObj, err := h.frostfs.GetObject(ctx, frostfs.PrmObjectGet{ + PrmAuth: frostfs.PrmAuth{BearerToken: bearerToken(ctx)}, + Address: p.objInfo.Address(), + }) + if err != nil { + return nil, fmt.Errorf("get combined object '%s': %w", p.objInfo.ID.EncodeToString(), err) + } + + var parts []*data.PartInfo + if err = json.NewDecoder(combinedObj.Payload).Decode(&parts); err != nil { + return nil, fmt.Errorf("unmarshal combined object parts: %w", err) + } + + isEncrypted := FormEncryptionInfo(p.objInfo.Headers).Enabled + objParts := make([]frostfs.PartObj, len(parts)) + for i, part := range parts { + size := part.Size + if isEncrypted { + if size, err = sio.EncryptedSize(part.Size); err != nil { + return nil, fmt.Errorf("compute encrypted size: %w", err) + } + } + + objParts[i] = frostfs.PartObj{ + OID: part.OID, + Size: size, + } + } + + return frostfs.NewMultiObjectReader(ctx, frostfs.MultiObjectReaderConfig{ + Handler: h, + Off: p.off, + Ln: p.ln, + Parts: objParts, + BktInfo: p.bktInfo, + Log: h.log, + }) +} + +// InitFrostFSObjectPayloadReader initializes payload reader of the FrostFS object. +// Zero range corresponds to full payload (panics if only offset is set). +func (h *Handler) InitFrostFSObjectPayloadReader(ctx context.Context, p frostfs.GetFrostFSParams) (io.Reader, error) { + var prmAuth frostfs.PrmAuth + + var addr oid.Address + addr.SetContainer(p.BktInfo.CID) + addr.SetObject(p.Oid) + + if p.Off+p.Ln != 0 { + prm := frostfs.PrmObjectRange{ + PrmAuth: prmAuth, + Container: p.BktInfo.CID, + Object: p.Oid, + PayloadRange: [2]uint64{p.Off, p.Ln}, + Address: addr, + } + + return h.frostfs.RangeObject(ctx, prm) + } + + prm := frostfs.PrmObjectGet{ + PrmAuth: prmAuth, + Container: p.BktInfo.CID, + Object: p.Oid, + Address: addr, + } + + res, err := h.frostfs.GetObject(ctx, prm) + if err != nil { + return nil, err + } + + return res.Payload, nil +} + +// ObjectEncryption stores parsed object encryption headers. +type ObjectEncryption struct { + Enabled bool + Algorithm string + HMACKey string + HMACSalt string +} + +func FormEncryptionInfo(headers map[string]string) ObjectEncryption { + algorithm := headers[AttributeEncryptionAlgorithm] + return ObjectEncryption{ + Enabled: len(algorithm) > 0, + Algorithm: algorithm, + HMACKey: headers[AttributeHMACKey], + HMACSalt: headers[AttributeHMACSalt], + } +} diff --git a/internal/handler/reader.go b/internal/handler/reader.go index 65d258b..d3803d9 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -5,11 +5,12 @@ import ( "context" "io" "net/http" - "path" "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -47,19 +48,24 @@ func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (str return http.DetectContentType(buf), buf, err // to not lose io.EOF } -func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oid.Address) { +type getPayloadParams struct { + obj *frostfs.Object + req request + bktinfo *data.BucketInfo + attrs map[string]string +} + +func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.Address, bktInfo *data.BucketInfo) { var ( - err error - dis = "inline" - start = time.Now() - filename string + shouldDownload = req.QueryArgs().GetBool("download") + start = time.Now() ) - prm := PrmObjectGet{ - PrmAuth: PrmAuth{ + prm := frostfs.PrmObjectGet{ + PrmAuth: frostfs.PrmAuth{ BearerToken: bearerToken(ctx), }, - Address: objectAddress, + Address: objAddress, } rObj, err := h.frostfs.GetObject(ctx, prm) @@ -70,51 +76,40 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi // we can't close reader in this function, so how to do it? - if req.Request.URI().QueryArgs().GetBool("download") { - dis = "attachment" + attrs := makeAttributesMap(rObj.Header.Attributes()) + req.setAttributes(attrs) + + req.setIDs(rObj.Header) + req.setDisposition(shouldDownload, attrs[object.AttributeFileName]) + + if err = req.setTimestamp(attrs[object.AttributeTimestamp]); err != nil { + req.log.Error(logs.CouldntParseCreationDate, + zap.String("val", attrs[object.AttributeTimestamp]), + zap.Error(err)) + response.Error(req.RequestCtx, "failed to convert timestamp: "+err.Error(), fasthttp.StatusInternalServerError) } - payloadSize := rObj.Header.PayloadSize() + payloadParams := getPayloadParams{ + obj: rObj, + req: req, + bktinfo: bktInfo, + attrs: attrs, + } + payload, payloadSize, err := h.getPayload(payloadParams) + if err != nil { + req.handleFrostFSErr(err, start) + return + } req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) - var contentType string - for _, attr := range rObj.Header.Attributes() { - key := attr.Key() - val := attr.Value() - if !isValidToken(key) || !isValidValue(val) { - continue - } - - key = utils.BackwardTransformIfSystem(key) - - req.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) - switch key { - case object.AttributeFileName: - filename = val - case object.AttributeTimestamp: - value, err := strconv.ParseInt(val, 10, 64) - if err != nil { - req.log.Info(logs.CouldntParseCreationDate, - zap.String("key", key), - zap.String("val", val), - zap.Error(err)) - continue - } - req.Response.Header.Set(fasthttp.HeaderLastModified, - time.Unix(value, 0).UTC().Format(http.TimeFormat)) - case object.AttributeContentType: - contentType = val - } - } - - idsToResponse(&req.Response, &rObj.Header) + contentType := attrs[object.AttributeContentType] if len(contentType) == 0 { // determine the Content-Type from the payload head var payloadHead []byte contentType, payloadHead, err = readContentType(payloadSize, func(uint64) (io.Reader, error) { - return rObj.Payload, nil + return payload, nil }) if err != nil && err != io.EOF { req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) @@ -126,16 +121,27 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi var headReader io.Reader = bytes.NewReader(payloadHead) if err != io.EOF { // otherwise, we've already read full payload - headReader = io.MultiReader(headReader, rObj.Payload) + headReader = io.MultiReader(headReader, payload) } // note: we could do with io.Reader, but SetBodyStream below closes body stream // if it implements io.Closer and that's useful for us. - rObj.Payload = readCloser{headReader, rObj.Payload} + payload = readCloser{headReader, payload} } req.SetContentType(contentType) - req.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) - - req.Response.SetBodyStream(rObj.Payload, int(payloadSize)) + req.Response.SetBodyStream(payload, int(payloadSize)) +} + +func makeAttributesMap(attrs []object.Attribute) map[string]string { + attributes := make(map[string]string) + for _, attr := range attrs { + if !isValidToken(attr.Key()) || !isValidValue(attr.Value()) { + continue + } + key := utils.BackwardTransformIfSystem(attr.Key()) + + attributes[key] = attr.Value() + } + return attributes } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index cea2250..e565b79 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -9,6 +9,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" @@ -131,8 +132,8 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { obj.SetOwnerID(*h.ownerID) obj.SetAttributes(attributes...) - prm := PrmObjectCreate{ - PrmAuth: PrmAuth{ + prm := frostfs.PrmObjectCreate{ + PrmAuth: frostfs.PrmAuth{ BearerToken: h.fetchBearerToken(ctx), }, Object: obj, diff --git a/internal/handler/utils.go b/internal/handler/utils.go index a5a53ed..e322cc5 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -2,14 +2,19 @@ package handler import ( "context" + "net/http" + "path" + "strconv" "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -31,6 +36,43 @@ func (r *request) handleFrostFSErr(err error, start time.Time) { response.Error(r.RequestCtx, msg, statusCode) } +func (r *request) setAttributes(attrs map[string]string) { + for key, val := range attrs { + r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) + } +} + +func (r *request) setIDs(obj object.Object) { + objID, _ := obj.ID() + cnrID, _ := obj.ContainerID() + r.Response.Header.Set(hdrObjectID, objID.String()) + r.Response.Header.Set(hdrOwnerID, obj.OwnerID().String()) + r.Response.Header.Set(hdrContainerID, cnrID.String()) +} + +func (r *request) setDisposition(shouldDownload bool, filename string) { + const ( + inlineDisposition = "inline" + attachmentDisposition = "attachment" + ) + dis := inlineDisposition + if shouldDownload { + dis = attachmentDisposition + } + r.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) +} + +func (r *request) setTimestamp(timestamp string) error { + value, err := strconv.ParseInt(timestamp, 10, 64) + if err != nil { + return err + } + r.Response.Header.Set(fasthttp.HeaderLastModified, + time.Unix(value, 0).UTC().Format(http.TimeFormat)) + + return nil +} + func bearerToken(ctx context.Context) *bearer.Token { if tkn, err := tokens.LoadBearerToken(ctx); err == nil { return tkn diff --git a/internal/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go similarity index 62% rename from internal/frostfs/frostfs.go rename to internal/service/frostfs/frostfs.go index 2610564..e1ea3c5 100644 --- a/internal/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -7,15 +7,115 @@ import ( "io" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" +) + +type ResObjectSearch interface { + Read(buf []oid.ID) (int, error) + Iterate(f func(oid.ID) bool) error + Close() +} + +// Object represents FrostFS object. +type Object struct { + // Object header (doesn't contain payload). + Header object.Object + + // Object payload part encapsulated in io.Reader primitive. + // Returns ErrAccessDenied on read access violation. + Payload io.ReadCloser +} + +type ( + // PrmContainer groups parameters of FrostFS.Container operation. + PrmContainer struct { + // Container identifier. + ContainerID cid.ID + } + // PrmAuth groups authentication parameters for the FrostFS operation. + PrmAuth struct { + // Bearer token to be used for the operation. Overlaps PrivateKey. Optional. + BearerToken *bearer.Token + } + // PrmObjectHead groups parameters of FrostFS.HeadObject operation. + PrmObjectHead struct { + // Authentication parameters. + PrmAuth + + // Address to read the object header from. + Address oid.Address + } + // PrmObjectGet groups parameters of FrostFS.GetObject operation. + PrmObjectGet struct { + // Authentication parameters. + PrmAuth + + // Address to read the object header from. + Address oid.Address + Container cid.ID + Object oid.ID + } + // PrmObjectRange groups parameters of FrostFS.RangeObject operation. + PrmObjectRange struct { + // Authentication parameters. + PrmAuth + + // Address to read the object header from. + Address oid.Address + + // Offset-length range of the object payload to be read. + PayloadRange [2]uint64 + + Container cid.ID + + Object oid.ID + } + // PrmObjectCreate groups parameters of FrostFS.CreateObject operation. + PrmObjectCreate struct { + // Authentication parameters. + PrmAuth + + Object *object.Object + + // Object payload encapsulated in io.Reader primitive. + Payload io.Reader + + // Enables client side object preparing. + ClientCut bool + + // Disables using Tillich-Zémor hash for payload. + WithoutHomomorphicHash bool + + // Sets max buffer size to read payload. + BufferMaxSize uint64 + } + // PrmObjectSearch groups parameters of FrostFS.sear SearchObjects operation. + PrmObjectSearch struct { + // Authentication parameters. + PrmAuth + + // Container to select the objects from. + Container cid.ID + + Filters object.SearchFilters + } +) + +var ( + // ErrAccessDenied is returned from FrostFS in case of access violation. + ErrAccessDenied = errors.New("access denied") + // ErrGatewayTimeout is returned from FrostFS in case of timeout, deadline exceeded etc. + ErrGatewayTimeout = errors.New("gateway timeout") ) // FrostFS represents virtual connection to the FrostFS network. @@ -33,9 +133,9 @@ func NewFrostFS(p *pool.Pool) *FrostFS { } // Container implements frostfs.FrostFS interface method. -func (x *FrostFS) Container(ctx context.Context, layerPrm handler.PrmContainer) (*container.Container, error) { +func (x *FrostFS) Container(ctx context.Context, containerPrm PrmContainer) (*container.Container, error) { prm := pool.PrmContainerGet{ - ContainerID: layerPrm.ContainerID, + ContainerID: containerPrm.ContainerID, } res, err := x.pool.GetContainer(ctx, prm) @@ -47,7 +147,7 @@ func (x *FrostFS) Container(ctx context.Context, layerPrm handler.PrmContainer) } // CreateObject implements frostfs.FrostFS interface method. -func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) (oid.ID, error) { +func (x *FrostFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.ID, error) { var prmPut pool.PrmObjectPut prmPut.SetHeader(*prm.Object) prmPut.SetPayload(prm.Payload) @@ -78,7 +178,7 @@ func (x payloadReader) Read(p []byte) (int, error) { } // HeadObject implements frostfs.FrostFS interface method. -func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*object.Object, error) { +func (x *FrostFS) HeadObject(ctx context.Context, prm PrmObjectHead) (*object.Object, error) { var prmHead pool.PrmObjectHead prmHead.SetAddress(prm.Address) @@ -95,7 +195,7 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o } // GetObject implements frostfs.FrostFS interface method. -func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*handler.Object, error) { +func (x *FrostFS) GetObject(ctx context.Context, prm PrmObjectGet) (*Object, error) { var prmGet pool.PrmObjectGet prmGet.SetAddress(prm.Address) @@ -108,14 +208,14 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han return nil, handleObjectError("init full object reading via connection pool", err) } - return &handler.Object{ + return &Object{ Header: res.Header, Payload: res.Payload, }, nil } // RangeObject implements frostfs.FrostFS interface method. -func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) (io.ReadCloser, error) { +func (x *FrostFS) RangeObject(ctx context.Context, prm PrmObjectRange) (io.ReadCloser, error) { var prmRange pool.PrmObjectRange prmRange.SetAddress(prm.Address) prmRange.SetOffset(prm.PayloadRange[0]) @@ -134,7 +234,7 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) ( } // SearchObjects implements frostfs.FrostFS interface method. -func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch) (handler.ResObjectSearch, error) { +func (x *FrostFS) SearchObjects(ctx context.Context, prm PrmObjectSearch) (ResObjectSearch, error) { var prmSearch pool.PrmObjectSearch prmSearch.SetContainerID(prm.Container) prmSearch.SetFilters(prm.Filters) @@ -202,11 +302,11 @@ func handleObjectError(msg string, err error) error { } if reason, ok := IsErrObjectAccessDenied(err); ok { - return fmt.Errorf("%s: %w: %s", msg, handler.ErrAccessDenied, reason) + return fmt.Errorf("%s: %w: %s", msg, ErrAccessDenied, reason) } if IsTimeoutError(err) { - return fmt.Errorf("%s: %w: %s", msg, handler.ErrGatewayTimeout, err.Error()) + return fmt.Errorf("%s: %w: %s", msg, ErrGatewayTimeout, err.Error()) } return fmt.Errorf("%s: %w", msg, err) diff --git a/internal/service/frostfs/multi_object_reader.go b/internal/service/frostfs/multi_object_reader.go new file mode 100644 index 0000000..a3cda9d --- /dev/null +++ b/internal/service/frostfs/multi_object_reader.go @@ -0,0 +1,168 @@ +package frostfs + +import ( + "context" + "errors" + "fmt" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" +) + +type GetFrostFSParams struct { + // payload range + Off, Ln uint64 + + Oid oid.ID + BktInfo *data.BucketInfo +} + +type PartObj struct { + OID oid.ID + Size uint64 +} + +type readerInitiator interface { + InitFrostFSObjectPayloadReader(ctx context.Context, p GetFrostFSParams) (io.Reader, error) +} + +// MultiObjectReader implements io.Reader of payloads of the object list stored in the FrostFS network. +type MultiObjectReader struct { + ctx context.Context + log *zap.Logger + + layer readerInitiator + + startPartOffset uint64 + endPartLength uint64 + + prm GetFrostFSParams + + curIndex int + curReader io.Reader + + parts []PartObj +} + +type MultiObjectReaderConfig struct { + Handler readerInitiator + Log *zap.Logger + + // the offset of complete object and total size to read + Off, Ln uint64 + + BktInfo *data.BucketInfo + Parts []PartObj +} + +var ( + errOffsetIsOutOfRange = errors.New("offset is out of payload range") + errLengthIsOutOfRange = errors.New("length is out of payload range") + errEmptyPartsList = errors.New("empty parts list") + errorZeroRangeLength = errors.New("zero range length") +) + +func NewMultiObjectReader(ctx context.Context, cfg MultiObjectReaderConfig) (*MultiObjectReader, error) { + if len(cfg.Parts) == 0 { + return nil, errEmptyPartsList + } + + r := &MultiObjectReader{ + ctx: ctx, + layer: cfg.Handler, + prm: GetFrostFSParams{ + BktInfo: cfg.BktInfo, + }, + parts: cfg.Parts, + log: cfg.Log, + } + + if cfg.Off+cfg.Ln == 0 { + return r, nil + } + + if cfg.Off > 0 && cfg.Ln == 0 { + return nil, errorZeroRangeLength + } + + startPartIndex, startPartOffset := findStartPart(cfg) + if startPartIndex == -1 { + return nil, errOffsetIsOutOfRange + } + r.startPartOffset = startPartOffset + + endPartIndex, endPartLength := findEndPart(cfg) + if endPartIndex == -1 { + return nil, errLengthIsOutOfRange + } + r.endPartLength = endPartLength + + r.parts = cfg.Parts[startPartIndex : endPartIndex+1] + + return r, nil +} + +func findStartPart(cfg MultiObjectReaderConfig) (index int, offset uint64) { + position := cfg.Off + for i, part := range cfg.Parts { + // Strict inequality when searching for start position to avoid reading zero length part. + if position < part.Size { + return i, position + } + position -= part.Size + } + + return -1, 0 +} + +func findEndPart(cfg MultiObjectReaderConfig) (index int, length uint64) { + position := cfg.Off + cfg.Ln + for i, part := range cfg.Parts { + // Non-strict inequality when searching for end position to avoid out of payload range error. + if position <= part.Size { + return i, position + } + position -= part.Size + } + + return -1, 0 +} + +func (x *MultiObjectReader) Read(p []byte) (n int, err error) { + if x.curReader != nil { + n, err = x.curReader.Read(p) + if !errors.Is(err, io.EOF) { + return n, err + } + x.curIndex++ + } + + if x.curIndex == len(x.parts) { + return n, io.EOF + } + + x.prm.Oid = x.parts[x.curIndex].OID + + if x.curIndex == 0 { + x.prm.Off = x.startPartOffset + x.prm.Ln = x.parts[x.curIndex].Size - x.startPartOffset + } + + if x.curIndex == len(x.parts)-1 { + x.prm.Ln = x.endPartLength - x.prm.Off + } + + x.curReader, err = x.layer.InitFrostFSObjectPayloadReader(x.ctx, x.prm) + if err != nil { + return n, fmt.Errorf("init payload reader for the next part: %w", err) + } + + x.prm.Off = 0 + x.prm.Ln = 0 + + next, err := x.Read(p[n:]) + + return n + next, err +} diff --git a/internal/service/frostfs/multi_object_reader_test.go b/internal/service/frostfs/multi_object_reader_test.go new file mode 100644 index 0000000..f1bf2bf --- /dev/null +++ b/internal/service/frostfs/multi_object_reader_test.go @@ -0,0 +1,137 @@ +package frostfs + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "testing" + + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +type readerInitiatorMock struct { + parts map[oid.ID][]byte +} + +func (r *readerInitiatorMock) InitFrostFSObjectPayloadReader(_ context.Context, p GetFrostFSParams) (io.Reader, error) { + partPayload, ok := r.parts[p.Oid] + if !ok { + return nil, errors.New("part not found") + } + + if p.Off+p.Ln == 0 { + return bytes.NewReader(partPayload), nil + } + + if p.Off > uint64(len(partPayload)-1) { + return nil, fmt.Errorf("invalid offset: %d/%d", p.Off, len(partPayload)) + } + + if p.Off+p.Ln > uint64(len(partPayload)) { + return nil, fmt.Errorf("invalid range: %d-%d/%d", p.Off, p.Off+p.Ln, len(partPayload)) + } + + return bytes.NewReader(partPayload[p.Off : p.Off+p.Ln]), nil +} + +func prepareDataReader() ([]byte, []PartObj, *readerInitiatorMock) { + mockInitReader := &readerInitiatorMock{ + parts: map[oid.ID][]byte{ + oidtest.ID(): []byte("first part 1"), + oidtest.ID(): []byte("second part 2"), + oidtest.ID(): []byte("third part 3"), + }, + } + + var fullPayload []byte + parts := make([]PartObj, 0, len(mockInitReader.parts)) + for id, payload := range mockInitReader.parts { + parts = append(parts, PartObj{OID: id, Size: uint64(len(payload))}) + fullPayload = append(fullPayload, payload...) + } + + return fullPayload, parts, mockInitReader +} + +func TestMultiReader(t *testing.T) { + ctx := context.Background() + + fullPayload, parts, mockInitReader := prepareDataReader() + + for _, tc := range []struct { + name string + off uint64 + ln uint64 + err error + }{ + { + name: "simple read all", + }, + { + name: "simple read with length", + ln: uint64(len(fullPayload)), + }, + { + name: "middle of parts", + off: parts[0].Size + 2, + ln: 4, + }, + { + name: "first and second", + off: parts[0].Size - 4, + ln: 8, + }, + { + name: "first and third", + off: parts[0].Size - 4, + ln: parts[1].Size + 8, + }, + { + name: "second part", + off: parts[0].Size, + ln: parts[1].Size, + }, + { + name: "second and third", + off: parts[0].Size, + ln: parts[1].Size + parts[2].Size, + }, + { + name: "offset out of range", + off: uint64(len(fullPayload) + 1), + ln: 1, + err: errOffsetIsOutOfRange, + }, + { + name: "zero length", + off: parts[1].Size + 1, + ln: 0, + err: errorZeroRangeLength, + }, + } { + t.Run(tc.name, func(t *testing.T) { + multiReader, err := NewMultiObjectReader(ctx, MultiObjectReaderConfig{ + Handler: mockInitReader, + Parts: parts, + Off: tc.off, + Ln: tc.ln, + }) + require.ErrorIs(t, err, tc.err) + + if tc.err == nil { + off := tc.off + ln := tc.ln + if off+ln == 0 { + ln = uint64(len(fullPayload)) + } + data, err := io.ReadAll(multiReader) + require.NoError(t, err) + require.Equal(t, fullPayload[off:off+ln], data) + } + }) + } +} diff --git a/internal/frostfs/services/pool_wrapper.go b/internal/service/frostfs/pool_wrapper.go similarity index 99% rename from internal/frostfs/services/pool_wrapper.go rename to internal/service/frostfs/pool_wrapper.go index f7b0a26..11565a6 100644 --- a/internal/frostfs/services/pool_wrapper.go +++ b/internal/service/frostfs/pool_wrapper.go @@ -1,4 +1,4 @@ -package services +package frostfs import ( "context"