From 339c3d934d0f2007d3c16d4fcca57b3e3ef943b0 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 13 Oct 2020 12:30:12 +0300 Subject: [PATCH 01/13] [#25] Upgrade dependencies closes #25 Signed-off-by: Evgeniy Kulikov --- go.mod | 20 ++++++++----- go.sum | 93 ++++++++++++++++++++++++---------------------------------- 2 files changed, 51 insertions(+), 62 deletions(-) diff --git a/go.mod b/go.mod index 33040ec5..688537ed 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,14 @@ module github.com/nspcc-dev/neofs-s3-gate go 1.14 require ( - github.com/awalterschulze/gographviz v2.0.1+incompatible // indirect - github.com/aws/aws-sdk-go v1.33.20 + github.com/aws/aws-sdk-go v1.27.0 github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/google/uuid v1.1.1 + github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.7.4 github.com/mitchellh/mapstructure v1.3.3 // indirect github.com/mr-tron/base58 v1.2.0 // indirect - github.com/nspcc-dev/hrw v1.0.9 // indirect - github.com/nspcc-dev/neofs-api-go v1.3.0 - github.com/nspcc-dev/neofs-authmate v0.0.0-20200806174343-d3b2deb75d54 + github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201007071636-fa18f5ede719 + github.com/nspcc-dev/neofs-authmate v0.0.0 github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/pelletier/go-toml v1.8.0 // indirect github.com/pkg/errors v0.9.1 @@ -24,12 +22,18 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 go.uber.org/atomic v1.6.0 - go.uber.org/zap v1.15.0 + go.uber.org/zap v1.16.0 golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect golang.org/x/sys v0.0.0-20200806125547-5acd03effb82 // indirect golang.org/x/text v0.3.3 // indirect google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 // indirect - google.golang.org/grpc v1.31.0 + google.golang.org/grpc v1.33.0 google.golang.org/protobuf v1.25.0 // indirect gopkg.in/ini.v1 v1.57.0 // indirect ) + +// temporary +replace ( + github.com/nspcc-dev/neofs-api-go => ../neofs-api + github.com/nspcc-dev/neofs-authmate => ../neofs-authmate +) diff --git a/go.sum b/go.sum index f8026272..0d8fa944 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,7 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/ github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -42,15 +43,9 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310 h1:t+qxRrRtwNiUYA+Xh2jSXhoG2grnMCMKX4Fg6lx9X1U= -github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= -github.com/awalterschulze/gographviz v2.0.1+incompatible h1:XIECBRq9VPEQqkQL5pw2OtjCAdrtIgFKoJU8eT98AS8= -github.com/awalterschulze/gographviz v2.0.1+incompatible/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.33.20 h1:mtXKHmMQO6o0i2GTjyiVNZGlXqJDCUbiik0OQeMds/o= -github.com/aws/aws-sdk-go v1.33.20/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -58,6 +53,15 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= @@ -89,9 +93,12 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -133,17 +140,14 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -185,6 +189,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -239,17 +245,16 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -257,8 +262,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -278,7 +283,6 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -309,8 +313,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= -github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -327,32 +329,20 @@ github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E= -github.com/nspcc-dev/hrw v1.0.1/go.mod h1:2gbvQ0nRi9+er+djibFyMLMWC8Ilwe7Z9pYCU6OrkMM= -github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= -github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= -github.com/nspcc-dev/neo-go v0.90.0 h1:ABNDrJuF9C1XuLQu0q9DKSVMlg9eQn/g6rX8Jbr31bo= -github.com/nspcc-dev/neo-go v0.90.0/go.mod h1:pPFdnApJwUSRAnpdiPBZl7I7jv0doDg5naecpSPK4+Q= -github.com/nspcc-dev/neofs-api-go v1.3.0 h1:w0wYIXzPJIElwhqahnQw/1NKiHxjRZKJhDUMSbEHmdk= -github.com/nspcc-dev/neofs-api-go v1.3.0/go.mod h1:NlCjqm//ZRXBNlxtrilLM1GgkRz0mv4V3pdX8OcGoLw= -github.com/nspcc-dev/neofs-authmate v0.0.0-20200806174343-d3b2deb75d54 h1:wJRGauG23mfZkjb5fcMCVHl4OJa4dqGgODw+C0ruubI= -github.com/nspcc-dev/neofs-authmate v0.0.0-20200806174343-d3b2deb75d54/go.mod h1:p8AbgUUTZ7J6yLR+AUkjNvYQ1Y4z+aHT6AJfBrfEHPQ= +github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= +github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= +github.com/nspcc-dev/neofs-api v1.2.0 h1:3LM5LpaPumUGAlxcLsaNp9RnKYXFgWevUxtVKTQvl7o= +github.com/nspcc-dev/neofs-api v2.0.0+incompatible h1:wov1GbEHV2fKCh4ix++kCrBXAt/M23g5JGF7U8BS9Sk= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/netmap v1.2.0/go.mod h1:9vt5vDTHP0BdhUluQek4iFfMv+pCv5b1Kg7eM6WyQl8= -github.com/nspcc-dev/netmap v1.7.0 h1:ak64xn/gPdgYw4tsqSSF7kAGQGbEpeuJEF3XwBX4L9Y= -github.com/nspcc-dev/netmap v1.7.0/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= -github.com/nspcc-dev/netmap-ql v1.2.0 h1:8JvICU63eOwmrMUJiASsKqLE2GJV17EJqbyNlzgvHJw= -github.com/nspcc-dev/netmap-ql v1.2.0/go.mod h1:n96T6CNDp94mR2tTPhCZHGCJqoQGESSDP+Fj/XOGES4= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= -github.com/nspcc-dev/tzhash v1.4.0 h1:RVIR+mxOBHl58CE99+DXtE31ylD5PEkZSoWqoj4fVjg= -github.com/nspcc-dev/tzhash v1.4.0/go.mod h1:Z8gp/VZbyWgPhaMp/KTmeoW5UTynp/N60g0jTtSzBws= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -400,8 +390,6 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= -github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -416,8 +404,6 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.11.1 h1:0ZISXCMRuCZcxF77aT1BXY5m74mX2vrGYl1dSwBI0Jo= github.com/prometheus/common v0.11.1/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= @@ -427,19 +413,20 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= -github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -474,8 +461,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -486,8 +471,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= @@ -496,10 +481,9 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -github.com/vito/go-parse v0.0.0-20160925004003-ca8122a7499f h1:OnQSWRX8bwsQbXG+LFvF+I946/yiKGZAJ5e+mjna0GA= -github.com/vito/go-parse v0.0.0-20160925004003-ca8122a7499f/go.mod h1:11YxjMj8TtS9ze2Na+Cue+JFyFE9ZW+6UCgBtw3LP6k= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -531,6 +515,9 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -540,9 +527,10 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -572,7 +560,6 @@ golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -582,7 +569,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -605,7 +591,6 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -625,7 +610,6 @@ golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -649,7 +633,6 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -715,10 +698,10 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.0 h1:IBKSUNL2uBS2DkJBncPP+TwT0sp9tgA8A75NjHt6umg= +google.golang.org/grpc v1.33.0/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -756,6 +739,8 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 65b5d6e3d2463175ec240c698fd81e6af13fa326 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 13 Oct 2020 12:31:23 +0300 Subject: [PATCH 02/13] [#25] Fixes around cmd/gate closes #25 Signed-off-by: Evgeniy Kulikov --- cmd/gate/app-settings.go | 57 +++++++++++-------- cmd/gate/app.go | 118 ++++++++++++++++++++++++--------------- cmd/gate/app_option.go | 21 +++++++ cmd/gate/main.go | 2 +- 4 files changed, 129 insertions(+), 69 deletions(-) create mode 100644 cmd/gate/app_option.go diff --git a/cmd/gate/app-settings.go b/cmd/gate/app-settings.go index 4e121e48..6518c43c 100644 --- a/cmd/gate/app-settings.go +++ b/cmd/gate/app-settings.go @@ -1,16 +1,20 @@ package main import ( + "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "fmt" "io" + "io/ioutil" "os" "strconv" "strings" "time" + "github.com/nspcc-dev/neofs-authmate/accessbox/hcs" + crypto "github.com/nspcc-dev/neofs-crypto" "github.com/nspcc-dev/neofs-s3-gate/api/pool" "github.com/nspcc-dev/neofs-s3-gate/auth" @@ -91,40 +95,47 @@ type empty int func (empty) Read([]byte) (int, error) { return 0, io.EOF } -func fetchAuthCenter(l *zap.Logger, v *viper.Viper, peers []pool.Peer) (*auth.Center, error) { +func fetchGateAuthKeys(v *viper.Viper) (*hcs.X25519Keys, error) { + path := v.GetString(cfgGateAuthPrivateKey) + + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + return hcs.NewKeys(data) +} + +func fetchNeoFSKey(v *viper.Viper) (*ecdsa.PrivateKey, error) { var ( - err error - neofsPrivateKey *ecdsa.PrivateKey + err error + key *ecdsa.PrivateKey ) - switch nfspk := v.GetString(cfgNeoFSPrivateKey); nfspk { + + switch val := v.GetString(cfgNeoFSPrivateKey); val { case generated: - neofsPrivateKey, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + key, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { return nil, errors.Wrap(err, "could not generate NeoFS private key") } default: - neofsPrivateKey, err = crypto.LoadPrivateKey(nfspk) + key, err = crypto.LoadPrivateKey(val) if err != nil { return nil, errors.Wrap(err, "could not load NeoFS private key") } } - gapk := v.GetString(cfgGateAuthPrivateKey) - gateAuthPrivateKey, err := auth.LoadGateAuthPrivateKey(gapk) - if err != nil { - return nil, errors.Wrapf(err, "could not load gate auth private key %q", gapk) - } - // NB: Maybe choose a peer more smarter. - center, err := auth.NewCenter(l, peers[0].Address) - if err != nil { - return nil, errors.Wrap(err, "failed to create auth center") - } - if err = center.SetAuthKeys(gateAuthPrivateKey); err != nil { - return nil, errors.Wrap(err, "failed to set gate auth keys") - } - if err = center.SetNeoFSKeys(neofsPrivateKey); err != nil { - return nil, errors.Wrap(err, "failed to set NeoFS keys") - } - return center, nil + + return key, nil +} + +func fetchAuthCenter(ctx context.Context, p *authCenterParams) (*auth.Center, error) { + return auth.New(ctx, &auth.Params{ + Con: p.Pool, + Log: p.Logger, + Timeout: p.Timeout, + GAKey: p.GateAuthKeys, + NFKey: p.NeoFSPrivateKey, + }) } func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.Peer { diff --git a/cmd/gate/app.go b/cmd/gate/app.go index 043d58e2..baf2cddf 100644 --- a/cmd/gate/app.go +++ b/cmd/gate/app.go @@ -7,6 +7,7 @@ import ( "net/http" "time" + "github.com/nspcc-dev/neofs-authmate/accessbox/hcs" "github.com/nspcc-dev/neofs-s3-gate/api" "github.com/nspcc-dev/neofs-s3-gate/api/handler" "github.com/nspcc-dev/neofs-s3-gate/api/layer" @@ -19,13 +20,13 @@ import ( type ( App struct { - center *auth.Center - cli pool.Pool - log *zap.Logger - cfg *viper.Viper - tls *tlsConfig - obj layer.Client - api api.Handler + cli pool.Pool + ctr *auth.Center + log *zap.Logger + cfg *viper.Viper + tls *tlsConfig + obj layer.Client + api api.Handler conTimeout time.Duration reqTimeout time.Duration @@ -44,14 +45,18 @@ type ( } ) -func newApp(l *zap.Logger, v *viper.Viper) *App { +func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App { var ( - err error - cli pool.Pool - tls *tlsConfig - caller api.Handler - obj layer.Client - key *ecdsa.PrivateKey + err error + cli pool.Pool + tls *tlsConfig + caller api.Handler + ctr *auth.Center + obj layer.Client + + gaKey *hcs.X25519Keys + nfKey *ecdsa.PrivateKey + reBalance = defaultRebalanceTimer conTimeout = defaultConnectTimeout reqTimeout = defaultRequestTimeout @@ -59,19 +64,6 @@ func newApp(l *zap.Logger, v *viper.Viper) *App { maxClientsCount = defaultMaxClientsCount maxClientsDeadline = defaultMaxClientsDeadline ) - peers := fetchPeers(l, v) - center, err := fetchAuthCenter(l, v, peers) - if err != nil { - l.Fatal("failed to initialize auth center", zap.Error(err)) - } - key = center.GetNeoFSPrivateKey() - - if v.IsSet(cfgTLSKeyFile) && v.IsSet(cfgTLSCertFile) { - tls = &tlsConfig{ - KeyFile: v.GetString(cfgTLSKeyFile), - CertFile: v.GetString(cfgTLSCertFile), - } - } if v := v.GetDuration(cfgConnectTimeout); v > 0 { conTimeout = v @@ -89,15 +81,36 @@ func newApp(l *zap.Logger, v *viper.Viper) *App { maxClientsDeadline = v } + if v := v.GetDuration(cfgRebalanceTimer); v > 0 { + reBalance = v + } + + if nfKey, err = fetchNeoFSKey(v); err != nil { + l.Fatal("could not load NeoFS private key") + } + + if gaKey, err = fetchGateAuthKeys(v); err != nil { + l.Fatal("could not load gate auth key") + } + + if v.IsSet(cfgTLSKeyFile) && v.IsSet(cfgTLSCertFile) { + tls = &tlsConfig{ + KeyFile: v.GetString(cfgTLSKeyFile), + CertFile: v.GetString(cfgTLSCertFile), + } + } + + peers := fetchPeers(l, v) + poolConfig := &pool.Config{ + ConnectTimeout: conTimeout, + RequestTimeout: reqTimeout, ConnectionTTL: v.GetDuration(cfgConnectionTTL), - ConnectTimeout: v.GetDuration(cfgConnectTimeout), - RequestTimeout: v.GetDuration(cfgRequestTimeout), Peers: peers, Logger: l, - PrivateKey: key, + PrivateKey: nfKey, GRPCLogger: gRPCLogger(l), GRPCVerbose: v.GetBool(cfgGRPCVerbose), @@ -105,27 +118,42 @@ func newApp(l *zap.Logger, v *viper.Viper) *App { ClientParameters: keepalive.ClientParameters{}, } - if v := v.GetDuration(cfgRebalanceTimer); v > 0 { - reBalance = v - } - if cli, err = pool.New(poolConfig); err != nil { l.Fatal("could not prepare pool connections", zap.Error(err)) } + { // prepare auth center + ctx, cancel := context.WithTimeout(ctx, conTimeout) + defer cancel() + + params := &authCenterParams{ + Logger: l, + Pool: cli, + + Timeout: conTimeout, + + GateAuthKeys: gaKey, + NeoFSPrivateKey: nfKey, + } + + if ctr, err = fetchAuthCenter(ctx, params); err != nil { + l.Fatal("failed to initialize auth center", zap.Error(err)) + } + } + { // should establish connection with NeoFS Storage Nodes - ctx, cancel := context.WithTimeout(context.Background(), conTimeout) + ctx, cancel := context.WithTimeout(ctx, conTimeout) defer cancel() cli.ReBalance(ctx) - if _, err = cli.GetConnection(ctx); err != nil { + if _, err = cli.Connection(ctx); err != nil { l.Fatal("could not establish connection", zap.Error(err)) } } - if obj, err = layer.NewLayer(l, cli, key); err != nil { + if obj, err = layer.NewLayer(l, cli, nfKey); err != nil { l.Fatal("could not prepare ObjectLayer", zap.Error(err)) } @@ -134,13 +162,13 @@ func newApp(l *zap.Logger, v *viper.Viper) *App { } return &App{ - center: center, - cli: cli, - log: l, - cfg: v, - obj: obj, - tls: tls, - api: caller, + ctr: ctr, + cli: cli, + log: l, + cfg: v, + obj: obj, + tls: tls, + api: caller, webDone: make(chan struct{}, 1), wrkDone: make(chan struct{}, 1), @@ -189,7 +217,7 @@ func (a *App) Server(ctx context.Context) { attachProfiler(router, a.cfg, a.log) // Attach S3 API: - api.Attach(router, a.maxClients, a.api, a.center, a.log) + api.Attach(router, a.maxClients, a.api, a.ctr, a.log) // Use mux.Router as http.Handler srv.Handler = router diff --git a/cmd/gate/app_option.go b/cmd/gate/app_option.go new file mode 100644 index 00000000..188a228f --- /dev/null +++ b/cmd/gate/app_option.go @@ -0,0 +1,21 @@ +package main + +import ( + "crypto/ecdsa" + "time" + + "github.com/nspcc-dev/neofs-authmate/accessbox/hcs" + "github.com/nspcc-dev/neofs-s3-gate/api/pool" + "go.uber.org/zap" +) + +type ( + authCenterParams struct { + Pool pool.Client + Logger *zap.Logger + Timeout time.Duration + + GateAuthKeys *hcs.X25519Keys + NeoFSPrivateKey *ecdsa.PrivateKey + } +) diff --git a/cmd/gate/main.go b/cmd/gate/main.go index a2b06b11..e49f090b 100644 --- a/cmd/gate/main.go +++ b/cmd/gate/main.go @@ -4,8 +4,8 @@ func main() { var ( v = newSettings() l = newLogger(v) - a = newApp(l, v) g = newGracefulContext(l) + a = newApp(g, l, v) ) go a.Server(g) From b346a779338af8b4add79aca265eac38d6923b47 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 13 Oct 2020 12:32:23 +0300 Subject: [PATCH 03/13] [#25] Migrate connection pool to NeoFS API v2 closes #25 Signed-off-by: Evgeniy Kulikov --- api/pool/pool.go | 66 +++++++++++++++------------ api/pool/session.go | 107 ++++++-------------------------------------- api/pool/v2.go | 36 +++++++++++++++ 3 files changed, 86 insertions(+), 123 deletions(-) create mode 100644 api/pool/v2.go diff --git a/api/pool/pool.go b/api/pool/pool.go index b9374268..091604cd 100644 --- a/api/pool/pool.go +++ b/api/pool/pool.go @@ -8,8 +8,9 @@ import ( "sync" "time" - "github.com/nspcc-dev/neofs-api-go/service" - "github.com/nspcc-dev/neofs-api-go/state" + "github.com/nspcc-dev/neofs-api-go/pkg/token" + + "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/pkg/errors" "go.uber.org/atomic" "go.uber.org/zap" @@ -30,8 +31,8 @@ type ( Client interface { Status() error - GetConnection(context.Context) (*grpc.ClientConn, error) - SessionToken(ctx context.Context, params *SessionParams) (*service.Token, error) + Connection(context.Context) (*grpc.ClientConn, error) + Token(context.Context, *grpc.ClientConn) (*token.SessionToken, error) } Pool interface { @@ -74,20 +75,27 @@ type ( currentIdx *atomic.Int32 currentConn *grpc.ClientConn - reqHealth *state.HealthRequest + ping Pinger *sync.Mutex nodes []*node keys []uint32 conns map[uint32][]*node - key *ecdsa.PrivateKey - tokens map[string]*service.Token + tokens map[string]*token.SessionToken + + oid *owner.ID + key *ecdsa.PrivateKey unhealthy *atomic.Error } + + Pinger interface { + Call(context.Context, *grpc.ClientConn) error + } ) var ( + errNoConnections = errors.New("no connections") errBootstrapping = errors.New("bootstrapping") errEmptyConnection = errors.New("empty connection") errNoHealthyConnections = errors.New("no active connections") @@ -101,7 +109,7 @@ func New(cfg *Config) (Pool, error) { keys: make([]uint32, 0), nodes: make([]*node, 0), conns: make(map[uint32][]*node), - tokens: make(map[string]*service.Token), + tokens: make(map[string]*token.SessionToken), currentIdx: atomic.NewInt32(-1), @@ -114,6 +122,14 @@ func New(cfg *Config) (Pool, error) { unhealthy: atomic.NewError(errBootstrapping), } + p.oid = owner.NewID() + wallet, err := owner.NEO3WalletFromPublicKey(&p.key.PublicKey) + if err != nil { + return nil, err + } + + p.oid.SetNeo3Wallet(wallet) + if cfg.GRPCVerbose { grpclog.SetLoggerV2(cfg.GRPCLogger) } @@ -123,11 +139,8 @@ func New(cfg *Config) (Pool, error) { rand.Seed(seed) cfg.Logger.Info("used random seed", zap.Int64("seed", seed)) - p.reqHealth = new(state.HealthRequest) - p.reqHealth.SetTTL(service.NonForwardingTTL) - - if err := service.SignRequestData(cfg.PrivateKey, p.reqHealth); err != nil { - return nil, errors.Wrap(err, "could not sign `HealthRequest`") + if p.ping, err = newV2Ping(cfg.PrivateKey); err != nil { + return nil, err } for i := range cfg.Peers { @@ -147,6 +160,10 @@ func New(cfg *Config) (Pool, error) { zap.Uint32("weight", p.nodes[i].weight)) } + if len(p.nodes) == 0 { + return nil, errNoConnections + } + return p, nil } @@ -174,12 +191,12 @@ func (p *pool) ReBalance(ctx context.Context) { defer func() { p.Unlock() - _, err := p.GetConnection(ctx) + _, err := p.Connection(ctx) p.unhealthy.Store(err) }() keys := make(map[uint32]struct{}) - tokens := make(map[string]*service.Token) + tokens := make(map[string]*token.SessionToken) for i := range p.nodes { var ( @@ -187,7 +204,7 @@ func (p *pool) ReBalance(ctx context.Context) { exists bool err error start = time.Now() - tkn *service.Token + tkn *token.SessionToken conn = p.nodes[i].conn weight = p.nodes[i].weight ) @@ -222,7 +239,7 @@ func (p *pool) ReBalance(ctx context.Context) { { // try to prepare token ctx, cancel := context.WithTimeout(ctx, p.reqTimeout) - tkn, err = generateToken(ctx, conn, p.key) + tkn, err = prepareToken(ctx, conn, p.key) cancel() } @@ -240,7 +257,7 @@ func (p *pool) ReBalance(ctx context.Context) { p.log.Debug("connected to node", zap.String("address", p.nodes[i].address)) } else if tkn, exists = p.tokens[conn.Target()]; exists { // token exists, ignore - } else if tkn, err = generateToken(ctx, conn, p.key); err != nil { + } else if tkn, err = prepareToken(ctx, conn, p.key); err != nil { p.log.Error("could not prepare session token", zap.String("address", p.nodes[i].address), zap.Error(err)) @@ -311,7 +328,7 @@ func (p *pool) ReBalance(ctx context.Context) { }) } -func (p *pool) GetConnection(ctx context.Context) (*grpc.ClientConn, error) { +func (p *pool) Connection(ctx context.Context) (*grpc.ClientConn, error) { p.Lock() defer p.Unlock() @@ -361,16 +378,7 @@ func (p *pool) isAlive(ctx context.Context, cur *grpc.ClientConn) error { ctx, cancel := context.WithTimeout(ctx, p.reqTimeout) defer cancel() - res, err := state.NewStatusClient(cur).HealthCheck(ctx, p.reqHealth) - if err != nil { - p.log.Warn("could not fetch health-check", zap.Error(err)) - - return err - } else if !res.Healthy { - return errors.New(res.Status) - } - - return nil + return p.ping.Call(ctx, cur) default: return errors.New(st.String()) } diff --git a/api/pool/session.go b/api/pool/session.go index a4fc3842..ba6594ef 100644 --- a/api/pool/session.go +++ b/api/pool/session.go @@ -5,119 +5,38 @@ import ( "crypto/ecdsa" "math" - "github.com/nspcc-dev/neofs-api-go/refs" - "github.com/nspcc-dev/neofs-api-go/service" - "github.com/nspcc-dev/neofs-api-go/session" - crypto "github.com/nspcc-dev/neofs-crypto" - "github.com/pkg/errors" + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/token" "google.golang.org/grpc" ) -type ( - queryParams struct { - key *ecdsa.PrivateKey - addr refs.Address - verb service.Token_Info_Verb - } - - SessionParams struct { - Addr refs.Address - Conn *grpc.ClientConn - Verb service.Token_Info_Verb - } -) - -func (p *pool) fetchToken(ctx context.Context, con *grpc.ClientConn) (*session.Token, error) { +// SessionToken returns session token for connection +func (p *pool) Token(ctx context.Context, conn *grpc.ClientConn) (*token.SessionToken, error) { p.Lock() defer p.Unlock() - // if we had token for current connection - return it - if tkn, ok := p.tokens[con.Target()]; ok { + if tkn, ok := p.tokens[conn.Target()]; ok && tkn != nil { return tkn, nil } - // try to generate token for connection - tkn, err := generateToken(ctx, con, p.key) + // prepare session token + tkn, err := prepareToken(ctx, conn, p.key) if err != nil { return nil, err } - p.tokens[con.Target()] = tkn + // save token for current connection + p.tokens[conn.Target()] = tkn + return tkn, nil } -// SessionToken returns session token for connection -func (p *pool) SessionToken(ctx context.Context, params *SessionParams) (*service.Token, error) { - var ( - err error - tkn *session.Token - ) - - if params.Conn == nil { - return nil, errors.New("empty connection") - } else if tkn, err = p.fetchToken(ctx, params.Conn); err != nil { - return nil, err - } - - return prepareToken(tkn, queryParams{ - key: p.key, - addr: params.Addr, - verb: params.Verb, - }) -} - // creates token using -func generateToken(ctx context.Context, con *grpc.ClientConn, key *ecdsa.PrivateKey) (*service.Token, error) { - owner, err := refs.NewOwnerID(&key.PublicKey) +func prepareToken(ctx context.Context, con *grpc.ClientConn, key *ecdsa.PrivateKey) (*token.SessionToken, error) { + cli, err := client.New(key, client.WithGRPCConnection(con)) if err != nil { return nil, err } - token := new(service.Token) - token.SetOwnerID(owner) - token.SetExpirationEpoch(math.MaxUint64) - token.SetOwnerKey(crypto.MarshalPublicKey(&key.PublicKey)) - - creator, err := session.NewGRPCCreator(con, key) - if err != nil { - return nil, err - } - - res, err := creator.Create(ctx, token) - if err != nil { - return nil, err - } - - token.SetID(res.GetID()) - token.SetSessionKey(res.GetSessionKey()) - - return token, nil -} - -func prepareToken(t *service.Token, p queryParams) (*service.Token, error) { - sig := make([]byte, len(t.Signature)) - copy(sig, t.Signature) - - token := &service.Token{ - Token_Info: service.Token_Info{ - ID: t.ID, - OwnerID: t.OwnerID, - Verb: t.Verb, - Address: t.Address, - TokenLifetime: t.TokenLifetime, - SessionKey: t.SessionKey, - OwnerKey: t.OwnerKey, - }, - Signature: sig, - } - - token.SetAddress(p.addr) - token.SetVerb(p.verb) - - err := service.AddSignatureWithKey(p.key, service.NewSignedSessionToken(token)) - if err != nil { - return nil, err - } - - return token, nil + return cli.CreateSession(ctx, math.MaxUint64) } diff --git a/api/pool/v2.go b/api/pool/v2.go new file mode 100644 index 00000000..8610bd46 --- /dev/null +++ b/api/pool/v2.go @@ -0,0 +1,36 @@ +package pool + +import ( + "context" + "crypto/ecdsa" + + "github.com/nspcc-dev/neofs-api-go/v2/client" + "github.com/nspcc-dev/neofs-api-go/v2/netmap" + "github.com/nspcc-dev/neofs-api-go/v2/signature" + "github.com/pkg/errors" + "google.golang.org/grpc" +) + +type v2Ping struct { + req *netmap.LocalNodeInfoRequest +} + +func newV2Ping(key *ecdsa.PrivateKey) (Pinger, error) { + req := new(netmap.LocalNodeInfoRequest) + + if err := signature.SignServiceMessage(key, req); err != nil { + return nil, errors.Wrap(err, "could not sign `PingRequest`") + } + + return &v2Ping{req: req}, nil +} + +func (v *v2Ping) Call(ctx context.Context, conn *grpc.ClientConn) error { + if cli, err := netmap.NewClient(netmap.WithGlobalOpts(client.WithGRPCConn(conn))); err != nil { + return err + } else if _, err := cli.LocalNodeInfo(ctx, v.req); err != nil { + return err + } + + return nil +} From 3b1fd968f84b378814f8d5e46b0901b7a208c8df Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 13 Oct 2020 12:33:33 +0300 Subject: [PATCH 04/13] [#25] Migrate auth package to NeoFS API v2 - upgrade to NeoFS API v2 - upgrade to new Authmate package closes #25 Signed-off-by: Evgeniy Kulikov --- auth/bearer-token.go | 8 +-- auth/center.go | 115 ++++++++++++++----------------------------- 2 files changed, 42 insertions(+), 81 deletions(-) diff --git a/auth/bearer-token.go b/auth/bearer-token.go index 33c25301..a7ef9865 100644 --- a/auth/bearer-token.go +++ b/auth/bearer-token.go @@ -3,7 +3,7 @@ package auth import ( "context" - "github.com/nspcc-dev/neofs-api-go/service" + "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/pkg/errors" ) @@ -12,12 +12,12 @@ type contextKey string const bearerTokenContextKey contextKey = "bearer-token" // GetBearerToken returns a bearer token embedded into a context or error, if any. -func GetBearerToken(ctx context.Context) (*service.BearerTokenMsg, error) { +func GetBearerToken(ctx context.Context) (*token.BearerToken, error) { bt := ctx.Value(bearerTokenContextKey) if bt == nil { return nil, errors.New("got nil bearer token") } - v, ok := bt.(*service.BearerTokenMsg) + v, ok := bt.(*token.BearerToken) if !ok { return nil, errors.Errorf("extracted unexpected type other than bearer token's: %T", v) } @@ -25,6 +25,6 @@ func GetBearerToken(ctx context.Context) (*service.BearerTokenMsg, error) { } // SetBearerToken return a context with embedded bearer token. -func SetBearerToken(ctx context.Context, bearerToken *service.BearerTokenMsg) context.Context { +func SetBearerToken(ctx context.Context, bearerToken *token.BearerToken) context.Context { return context.WithValue(ctx, bearerTokenContextKey, bearerToken) } diff --git a/auth/center.go b/auth/center.go index f47582a7..de14d1aa 100644 --- a/auth/center.go +++ b/auth/center.go @@ -11,71 +11,56 @@ import ( "strings" "time" - aws_credentials "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials" v4 "github.com/aws/aws-sdk-go/aws/signer/v4" - "github.com/nspcc-dev/neofs-api-go/refs" - "github.com/nspcc-dev/neofs-api-go/service" + "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-authmate/accessbox/hcs" - "github.com/nspcc-dev/neofs-authmate/credentials" - "github.com/nspcc-dev/neofs-authmate/gates" - manager "github.com/nspcc-dev/neofs-authmate/neofsmanager" + "github.com/nspcc-dev/neofs-authmate/agents/s3" + manager "github.com/nspcc-dev/neofs-authmate/manager/neofs" "github.com/pkg/errors" "go.uber.org/zap" ) var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/(?P[^/]*)/(?P[^/]+)/aws4_request,\s*SignedHeaders=(?P.+),\s*Signature=(?P.+)`) -const emptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855` +type ( + Center struct { + man *manager.Manager + reg *regexpSubmatcher -// Center is a central app's authentication/authorization management unit. -type Center struct { - log *zap.Logger - submatcher *regexpSubmatcher - neofsCredentials *credentials.Credentials - manager *manager.Manager - authKeys *hcs.X25519Keys -} + keys *hcs.X25519Keys + } + + Params struct { + Timeout time.Duration + + Log *zap.Logger + Con manager.Connector + + GAKey *hcs.X25519Keys + NFKey *ecdsa.PrivateKey + } +) + +// New creates an instance of AuthCenter. +func New(ctx context.Context, p *Params) (*Center, error) { + m, err := manager.New(ctx, + manager.WithKey(p.NFKey), + manager.WithLogger(p.Log), + manager.WithConnector(p.Con)) -// NewCenter creates an instance of AuthCenter. -func NewCenter(log *zap.Logger, neofsNodeAddress string) (*Center, error) { - m, err := manager.NewManager(neofsNodeAddress) if err != nil { return nil, err } return &Center{ - log: log, - submatcher: ®expSubmatcher{re: authorizationFieldRegexp}, - manager: m, + man: m, + reg: ®expSubmatcher{re: authorizationFieldRegexp}, + + keys: p.GAKey, }, nil } -func (center *Center) SetNeoFSKeys(key *ecdsa.PrivateKey) error { - creds, err := credentials.NewFromKey(key) - if err != nil { - return err - } - center.neofsCredentials = creds - return nil -} - -func (center *Center) GetNeoFSPrivateKey() *ecdsa.PrivateKey { - return center.neofsCredentials.Key() -} - -func (center *Center) GetOwnerID() refs.OwnerID { - return center.neofsCredentials.OwnerID() -} - -func (center *Center) SetAuthKeys(key hcs.X25519PrivateKey) error { - keys, err := hcs.NewKeys(key) - if err != nil { - return err - } - center.authKeys = keys - return nil -} - -func (center *Center) AuthenticationPassed(request *http.Request) (*service.BearerTokenMsg, error) { +func (center *Center) AuthenticationPassed(request *http.Request) (*token.BearerToken, error) { queryValues := request.URL.Query() if queryValues.Get("X-Amz-Algorithm") == "AWS4-HMAC-SHA256" { return nil, errors.New("pre-signed form of request is not supported") @@ -84,7 +69,7 @@ func (center *Center) AuthenticationPassed(request *http.Request) (*service.Bear if len(authHeaderField) != 1 { return nil, errors.New("unsupported request: wrong length of Authorization header field") } - sms1 := center.submatcher.getSubmatches(authHeaderField[0]) + sms1 := center.reg.getSubmatches(authHeaderField[0]) if len(sms1) != 7 { return nil, errors.New("bad Authorization header field") } @@ -97,7 +82,7 @@ func (center *Center) AuthenticationPassed(request *http.Request) (*service.Bear return nil, errors.Wrap(err, "failed to parse x-amz-date header field") } accessKeyID := fmt.Sprintf("%s/%s", sms1["access_key_id_cid"], sms1["access_key_id_oid"]) - bearerToken, secretAccessKey, err := center.fetchBearerToken(accessKeyID) + res, err := s3.NewAgent(center.man).ObtainSecret(request.Context(), center.keys, accessKeyID) if err != nil { return nil, errors.Wrap(err, "failed to fetch bearer token") } @@ -110,7 +95,7 @@ func (center *Center) AuthenticationPassed(request *http.Request) (*service.Bear } } } - awsCreds := aws_credentials.NewStaticCredentials(accessKeyID, secretAccessKey, "") + awsCreds := credentials.NewStaticCredentials(accessKeyID, res.SecretAccessKey, "") signer := v4.NewSigner(awsCreds) body, err := readAndKeepBody(request) if err != nil { @@ -120,31 +105,11 @@ func (center *Center) AuthenticationPassed(request *http.Request) (*service.Bear if err != nil { return nil, errors.Wrap(err, "failed to sign temporary HTTP request") } - sms2 := center.submatcher.getSubmatches(otherRequest.Header.Get("Authorization")) + sms2 := center.reg.getSubmatches(otherRequest.Header.Get("Authorization")) if sms1["v4_signature"] != sms2["v4_signature"] { return nil, errors.Wrap(err, "failed to pass authentication procedure") } - return bearerToken, nil -} - -func (center *Center) fetchBearerToken(accessKeyID string) (*service.BearerTokenMsg, string, error) { - akid := new(refs.Address) - if err := akid.Parse(accessKeyID); err != nil { - return nil, "", errors.Wrap(err, "failed to parse access key id as refs.Address") - } - config := &gates.ObtainingConfig{ - BaseConfig: gates.BaseConfig{ - OperationalCredentials: center.neofsCredentials, - Manager: center.manager, - }, - GateKeys: center.authKeys, - SecretAddress: akid, - } - res, err := gates.ObtainSecret(config) - if err != nil { - return nil, "", errors.Wrap(err, "failed to obtain secret") - } - return res.BearerToken, res.SecretAccessKey, nil + return res.BearerToken, nil } // TODO: Make this write into a smart buffer backed by a file on a fast drive. @@ -160,7 +125,3 @@ func readAndKeepBody(request *http.Request) (*bytes.Reader, error) { request.Body = ioutil.NopCloser(bytes.NewReader(payload)) return bytes.NewReader(payload), nil } - -func LoadGateAuthPrivateKey(path string) (hcs.X25519PrivateKey, error) { - return ioutil.ReadFile(path) -} From 86e07c888245d2c0de4be49a5d5728106b87d5e4 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 13 Oct 2020 12:43:24 +0300 Subject: [PATCH 05/13] [#25] Fix linter warnings closes #25 Signed-off-by: Evgeniy Kulikov --- api/pool/pool.go | 3 +-- auth/errors.go | 17 ----------------- cmd/gate/app-logger.go | 3 +-- cmd/gate/app-settings.go | 1 - 4 files changed, 2 insertions(+), 22 deletions(-) delete mode 100644 auth/errors.go diff --git a/api/pool/pool.go b/api/pool/pool.go index 091604cd..dbaa1262 100644 --- a/api/pool/pool.go +++ b/api/pool/pool.go @@ -8,9 +8,8 @@ import ( "sync" "time" - "github.com/nspcc-dev/neofs-api-go/pkg/token" - "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/pkg/errors" "go.uber.org/atomic" "go.uber.org/zap" diff --git a/auth/errors.go b/auth/errors.go deleted file mode 100644 index 4a0dab58..00000000 --- a/auth/errors.go +++ /dev/null @@ -1,17 +0,0 @@ -package auth - -import "fmt" - -const ( - // Minimum length for MinIO access key. - accessKeyMinLen = 3 - - // Minimum length for MinIO secret key for both server and gateway mode. - secretKeyMinLen = 8 -) - -// Common errors generated for access and secret key validation. -var ( - ErrInvalidAccessKeyLength = fmt.Errorf("access key must be minimum %v or more characters long", accessKeyMinLen) - ErrInvalidSecretKeyLength = fmt.Errorf("secret key must be minimum %v or more characters long", secretKeyMinLen) -) diff --git a/cmd/gate/app-logger.go b/cmd/gate/app-logger.go index b3ad7d40..fe772c2d 100644 --- a/cmd/gate/app-logger.go +++ b/cmd/gate/app-logger.go @@ -3,11 +3,10 @@ package main import ( "strings" - "google.golang.org/grpc/grpclog" - "github.com/spf13/viper" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "google.golang.org/grpc/grpclog" ) type ( diff --git a/cmd/gate/app-settings.go b/cmd/gate/app-settings.go index 6518c43c..eb924911 100644 --- a/cmd/gate/app-settings.go +++ b/cmd/gate/app-settings.go @@ -14,7 +14,6 @@ import ( "time" "github.com/nspcc-dev/neofs-authmate/accessbox/hcs" - crypto "github.com/nspcc-dev/neofs-crypto" "github.com/nspcc-dev/neofs-s3-gate/api/pool" "github.com/nspcc-dev/neofs-s3-gate/auth" From ec56a1818e1d13986dc9df9cdce7eeb54e925882 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 19 Oct 2020 03:59:05 +0300 Subject: [PATCH 06/13] [#25] Upgrade dependencies closes #25 Signed-off-by: Evgeniy Kulikov --- go.mod | 4 +++- go.sum | 21 +++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 688537ed..b182151c 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.7.4 github.com/mitchellh/mapstructure v1.3.3 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201007071636-fa18f5ede719 github.com/nspcc-dev/neofs-authmate v0.0.0 github.com/nspcc-dev/neofs-crypto v0.3.0 @@ -21,11 +20,14 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 + github.com/stretchr/testify v1.6.1 go.uber.org/atomic v1.6.0 go.uber.org/zap v1.16.0 + golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect golang.org/x/sys v0.0.0-20200806125547-5acd03effb82 // indirect golang.org/x/text v0.3.3 // indirect + golang.org/x/tools v0.0.0-20200123022218-593de606220b // indirect google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 // indirect google.golang.org/grpc v1.33.0 google.golang.org/protobuf v1.25.0 // indirect diff --git a/go.sum b/go.sum index 0d8fa944..c62c233a 100644 --- a/go.sum +++ b/go.sum @@ -18,12 +18,14 @@ github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1 github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/Workiva/go-datastructures v1.0.50 h1:slDmfW6KCHcC7U+LP3DDBbm4fqTwZGn1beOFPfGaLvo= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= @@ -102,13 +104,17 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI= github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU= github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -138,6 +144,7 @@ github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgO github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-redis/redis v6.10.2+incompatible h1:SLbqrO/Ik1nhXA5/cbEs1P5MUBo1Qq4ihlNfGnnipPw= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -169,6 +176,7 @@ github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -334,8 +342,6 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neofs-api v1.2.0 h1:3LM5LpaPumUGAlxcLsaNp9RnKYXFgWevUxtVKTQvl7o= -github.com/nspcc-dev/neofs-api v2.0.0+incompatible h1:wov1GbEHV2fKCh4ix++kCrBXAt/M23g5JGF7U8BS9Sk= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= @@ -475,6 +481,7 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73 h1:I2drr5K0tykBofr74ZEGliE/Hf6fNkEbcPyFvsy7wZk= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= @@ -492,6 +499,7 @@ github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBU go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -513,8 +521,6 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEa go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -546,6 +552,8 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -652,8 +660,11 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114 h1:DnSr2mCsxyCE6ZgIkmcWUQY2R5cH/6wL7eIxEmQOMSE= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200123022218-593de606220b h1:ztSlcncMErSAUzXwnVO1iTPxHwtvOHBB26SGiyYXIEE= +golang.org/x/tools v0.0.0-20200123022218-593de606220b/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -698,8 +709,6 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.0 h1:IBKSUNL2uBS2DkJBncPP+TwT0sp9tgA8A75NjHt6umg= google.golang.org/grpc v1.33.0/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= From 7deeb68d47998ef36ec4c2c7cc3d061ff3f43271 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 19 Oct 2020 04:04:37 +0300 Subject: [PATCH 07/13] [#25] Migrate layer to NeoFS API v2 Naive migration, without any tries to start application closes #25 Signed-off-by: Evgeniy Kulikov --- api/layer/container.go | 72 ++--- api/layer/layer.go | 280 ++++++++---------- api/layer/object.go | 619 ++++++++------------------------------- api/layer/util.go | 61 ++-- api/layer/writer.go | 49 ++++ api/layer/writer_test.go | 77 +++++ 6 files changed, 420 insertions(+), 738 deletions(-) create mode 100644 api/layer/writer.go create mode 100644 api/layer/writer_test.go diff --git a/api/layer/container.go b/api/layer/container.go index 550ae05e..c4ea524b 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -4,9 +4,8 @@ import ( "context" "time" - "github.com/nspcc-dev/neofs-api-go/container" - "github.com/nspcc-dev/neofs-api-go/refs" - "github.com/nspcc-dev/neofs-api-go/service" + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-s3-gate/api" "github.com/nspcc-dev/neofs-s3-gate/auth" "go.uber.org/zap" @@ -15,7 +14,7 @@ import ( type ( BucketInfo struct { Name string - CID refs.CID + CID *container.ID Created time.Time } @@ -28,47 +27,36 @@ type ( } ) -func (n *layer) containerInfo(ctx context.Context, cid refs.CID) (*BucketInfo, error) { +func (n *layer) containerInfo(ctx context.Context, cid *container.ID) (*BucketInfo, error) { rid := api.GetRequestID(ctx) bearer, err := auth.GetBearerToken(ctx) if err != nil { n.log.Error("could not receive bearer token", + zap.Stringer("cid", cid), zap.String("request_id", rid), zap.Error(err)) return nil, err } - req := new(container.GetRequest) - req.SetCID(cid) - req.SetTTL(service.SingleForwardingTTL) - // req.SetBearer(bearer) - _ = bearer - if err = service.SignRequestData(n.key, req); err != nil { - n.log.Error("could not prepare request", - zap.String("request_id", rid), - zap.Error(err)) - return nil, err - } - - conn, err := n.cli.GetConnection(ctx) + cli, tkn, err := n.prepareClient(ctx) if err != nil { n.log.Error("could not prepare client", + zap.Stringer("cid", cid), zap.String("request_id", rid), zap.Error(err)) + return nil, err } - // todo: think about timeout - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - res, err := container.NewServiceClient(conn).Get(ctx, req) + res, err := cli.GetContainer(ctx, cid, client.WithSession(tkn)) if err != nil { - n.log.Error("could not list buckets", + n.log.Error("could not fetch container", + zap.Stringer("cid", cid), zap.String("request_id", rid), zap.Error(err)) + return nil, err } @@ -76,8 +64,8 @@ func (n *layer) containerInfo(ctx context.Context, cid refs.CID) (*BucketInfo, e return &BucketInfo{ CID: cid, - Name: cid.String(), // should be fetched from container.GetResponse - Created: time.Time{}, // should be fetched from container.GetResponse + Name: cid.String(), // should be fetched from container.Attributes + Created: time.Time{}, // should be fetched from container.Attributes }, nil } @@ -91,21 +79,9 @@ func (n *layer) containerList(ctx context.Context) ([]BucketInfo, error) { return nil, err } - req := new(container.ListRequest) - req.OwnerID = n.uid - req.SetTTL(service.SingleForwardingTTL) - // req.SetBearer(bearer) - _ = bearer - if err := service.SignRequestData(n.key, req); err != nil { - n.log.Error("could not prepare request", - zap.String("request_id", rid), - zap.Error(err)) - return nil, err - } - - conn, err := n.cli.GetConnection(ctx) + cli, tkn, err := n.prepareClient(ctx) if err != nil { n.log.Error("could not prepare client", zap.String("request_id", rid), @@ -113,20 +89,24 @@ func (n *layer) containerList(ctx context.Context) ([]BucketInfo, error) { return nil, err } - // todo: think about timeout - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() + // own, err := GetOwnerID(bearer) + // if err != nil { + // n.log.Error("could not fetch owner id", + // zap.String("request_id", rid), + // zap.Error(err)) + // return nil, err + // } - res, err := container.NewServiceClient(conn).List(ctx, req) + res, err := cli.ListContainers(ctx, tkn.OwnerID(), client.WithSession(tkn)) if err != nil { - n.log.Error("could not list buckets", + n.log.Error("could not fetch container", zap.String("request_id", rid), zap.Error(err)) return nil, err } - list := make([]BucketInfo, 0, len(res.CID)) - for _, cid := range res.CID { + list := make([]BucketInfo, 0, len(res)) + for _, cid := range res { info, err := n.containerInfo(ctx, cid) if err != nil { n.log.Error("could not fetch container info", diff --git a/api/layer/layer.go b/api/layer/layer.go index 0cb2c702..a8073aef 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -3,13 +3,17 @@ package layer import ( "context" "crypto/ecdsa" + "errors" "io" "strings" "time" - "github.com/nspcc-dev/neofs-api-go/object" - "github.com/nspcc-dev/neofs-api-go/refs" - "github.com/nspcc-dev/neofs-api-go/service" + "github.com/nspcc-dev/neofs-api-go/pkg" + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/container" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-s3-gate/api" "github.com/nspcc-dev/neofs-s3-gate/api/pool" "go.uber.org/zap" @@ -19,10 +23,19 @@ import ( type ( layer struct { + uid *owner.ID log *zap.Logger cli pool.Client - uid refs.OwnerID key *ecdsa.PrivateKey + + reqTimeout time.Duration + } + + Params struct { + Pool pool.Client + Logger *zap.Logger + Timeout time.Duration + NFKey *ecdsa.PrivateKey } GetObjectParams struct { @@ -50,7 +63,7 @@ type ( } NeoFS interface { - Get(ctx context.Context, address refs.Address) (*object.Object, error) + Get(ctx context.Context, address *object.Address) (*object.Object, error) } Client interface { @@ -73,61 +86,41 @@ type ( } ) -// AWS3NameHeader key in the object neofs. +// AWS3NameHeader key in the object NeoFS. const AWS3NameHeader = "filename" // NewGatewayLayer creates instance of layer. It checks credentials // and establishes gRPC connection with node. -func NewLayer(log *zap.Logger, cli pool.Client, key *ecdsa.PrivateKey) (Client, error) { - uid, err := refs.NewOwnerID(&key.PublicKey) +func NewLayer(p *Params) (Client, error) { + wallet, err := owner.NEO3WalletFromPublicKey(&p.NFKey.PublicKey) if err != nil { return nil, err } + + uid := owner.NewID() + uid.SetNeo3Wallet(wallet) + return &layer{ - cli: cli, - key: key, - log: log, uid: uid, + cli: p.Pool, + key: p.NFKey, + log: p.Logger, + + reqTimeout: p.Timeout, }, nil } // Get NeoFS Object by refs.Address (should be used by auth.Center) -func (n *layer) Get(ctx context.Context, address refs.Address) (*object.Object, error) { - conn, err := n.cli.GetConnection(ctx) +func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Object, error) { + cli, tkn, err := n.prepareClient(ctx) if err != nil { return nil, err } - token, err := n.cli.SessionToken(ctx, &pool.SessionParams{ - Conn: conn, - Addr: address, - Verb: service.Token_Info_Get, - }) + gop := new(client.GetObjectParams) + gop.WithAddress(address) - if err != nil { - return nil, err - } - - req := new(object.GetRequest) - req.Address = address - req.SetTTL(service.SingleForwardingTTL) - req.SetToken(token) - - err = service.SignRequestData(n.key, req) - if err != nil { - return nil, err - } - - // todo: think about timeout - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - cli, err := object.NewServiceClient(conn).Get(ctx, req) - if err != nil { - return nil, err - } - - return receiveObject(cli) + return cli.GetObject(ctx, gop, client.WithSession(tkn)) } // GetBucketInfo returns bucket name. @@ -160,31 +153,32 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj // pagination must be implemented with cache, because search results // may be different between search calls var ( + err error + bkt *BucketInfo + ids []*object.ID result ListObjectsInfo uniqNames = make(map[string]struct{}) ) - bkt, err := n.GetBucketInfo(ctx, p.Bucket) - if err != nil { + if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { + return nil, err + } else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil { return nil, err } - objectIDs, err := n.objectSearchContainer(ctx, bkt.CID) - if err != nil { - return nil, err - } - - ln := len(objectIDs) + ln := len(ids) // todo: check what happens if there is more than maxKeys objects if ln > p.MaxKeys { - result.IsTruncated = true ln = p.MaxKeys + result.IsTruncated = true } - result.Objects = make([]ObjectInfo, 0, ln) + result.Objects = make([]*ObjectInfo, 0, ln) - for i := 0; i < ln; i++ { - addr := refs.Address{ObjectID: objectIDs[i], CID: bkt.CID} + for _, id := range ids { + addr := object.NewAddress() + addr.SetObjectID(id) + addr.SetContainerID(bkt.CID) meta, err := n.objectHead(ctx, addr) if err != nil { @@ -192,17 +186,17 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj continue } - // ignore tombstone objects - _, hdr := meta.LastHeader(object.HeaderType(object.TombstoneHdr)) - if hdr != nil { - continue - } + // // ignore tombstone objects + // _, hdr := meta.LastHeader(object.HeaderType(object.TombstoneHdr)) + // if hdr != nil { + // continue + // } // ignore storage group objects - _, hdr = meta.LastHeader(object.HeaderType(object.StorageGroupHdr)) - if hdr != nil { - continue - } + // _, hdr = meta.LastHeader(object.HeaderType(object.StorageGroupHdr)) + // if hdr != nil { + // continue + // } // dirs don't exist in neofs, gateway stores full path to the file // in object header, e.g. `filename`:`/this/is/path/file.txt` @@ -225,8 +219,8 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj oi = objectInfoFromMeta(meta) } else { // if there are sub-entities in tail - dir oi = &ObjectInfo{ - Owner: meta.SystemHeader.OwnerID, - Bucket: meta.SystemHeader.CID.String(), + Owner: meta.GetOwnerID(), + Bucket: bkt.Name, Name: tail[:ind+1], // dir MUST have slash symbol in the end // IsDir: true, } @@ -236,7 +230,7 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj if _, ok := uniqNames[oi.Name]; !ok { uniqNames[oi.Name] = struct{}{} - result.Objects = append(result.Objects, *oi) + result.Objects = append(result.Objects, oi) } } } @@ -246,98 +240,74 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj // GetObject from storage. func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { - cid, err := refs.CIDFromString(p.Bucket) - if err != nil { + var ( + err error + oid *object.ID + cid = container.NewID() + ) + + if err = cid.Parse(p.Bucket); err != nil { + return err + } else if oid, err = n.objectFindID(ctx, &findParams{cid: cid, key: p.Object}); err != nil { return err } - oid, err := n.objectFindID(ctx, cid, p.Object, false) - if err != nil { - return err - } + addr := object.NewAddress() + addr.SetObjectID(oid) + addr.SetContainerID(cid) - addr := refs.Address{ - ObjectID: oid, - CID: cid, - } - _, err = n.objectGet(ctx, getParams{ - addr: addr, - start: p.Offset, + _, err = n.objectGet(ctx, &getParams{ + Writer: p.Writer, + + addr: addr, + + offset: p.Offset, length: p.Length, - writer: p.Writer, }) return err } // GetObjectInfo returns meta information about the object. -func (n *layer) GetObjectInfo(ctx context.Context, bucketName, objectName string) (*ObjectInfo, error) { - var meta *object.Object - if cid, err := refs.CIDFromString(bucketName); err != nil { +func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string) (*ObjectInfo, error) { + var ( + err error + oid *object.ID + cid = container.NewID() + + meta *object.Object + ) + + if err = cid.Parse(bucketName); err != nil { return nil, err - } else if oid, err := n.objectFindID(ctx, cid, objectName, false); err != nil { - return nil, err - } else if meta, err = n.objectHead(ctx, refs.Address{CID: cid, ObjectID: oid}); err != nil { + } else if oid, err = n.objectFindID(ctx, &findParams{cid: cid, key: filename}); err != nil { return nil, err } + + addr := object.NewAddress() + addr.SetObjectID(oid) + addr.SetContainerID(cid) + + if meta, err = n.objectHead(ctx, addr); err != nil { + return nil, err + } + return objectInfoFromMeta(meta), nil } +func GetOwnerID(tkn *token.BearerToken) (*owner.ID, error) { + switch pkg.SDKVersion().GetMajor() { + case 2: + id := tkn.ToV2().GetBody().GetOwnerID() + return owner.NewIDFromV2(id), nil + default: + return nil, errors.New("unknown version") + } +} + // PutObject into storage. func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) { - cid, err := refs.CIDFromString(p.Bucket) - if err != nil { - return nil, err - } - - _, err = n.objectFindID(ctx, cid, p.Object, true) - if err == nil { - return nil, &api.ObjectAlreadyExists{ - Bucket: p.Bucket, - Object: p.Object, - } - } - - oid, err := refs.NewObjectID() - if err != nil { - return nil, err - } - - sgid, err := refs.NewSGID() - if err != nil { - return nil, err - } - - addr := refs.Address{ - ObjectID: oid, - CID: cid, - } - - meta, err := n.objectPut(ctx, putParams{ - addr: addr, - size: p.Size, - name: p.Object, - r: p.Reader, - userHeaders: p.Header, - }) - if err != nil { - return nil, err - } - - oi := objectInfoFromMeta(meta) - - // for every object create storage group, otherwise object will be deleted - addr.ObjectID = sgid - - _, err = n.storageGroupPut(ctx, sgParams{ - addr: addr, - objects: []refs.ObjectID{oid}, - }) - if err != nil { - return nil, err - } - - return oi, nil + return n.objectPut(ctx, p) } // CopyObject from one bucket into another bucket. @@ -374,28 +344,34 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInf } // DeleteObject removes all objects with passed nice name. -func (n *layer) DeleteObject(ctx context.Context, bucket, object string) error { - cid, err := refs.CIDFromString(bucket) - if err != nil { - return &api.DeleteError{ - Err: err, - Object: object, - } - } +func (n *layer) DeleteObject(ctx context.Context, bucket, filename string) error { + var ( + err error + ids []*object.ID + cid = container.NewID() + ) - ids, err := n.objectFindIDs(ctx, cid, object) - if err != nil { + if err = cid.Parse(bucket); err != nil { return &api.DeleteError{ Err: err, - Object: object, + Object: filename, + } + } else if ids, err = n.objectSearch(ctx, &findParams{cid: cid, key: filename}); err != nil { + return &api.DeleteError{ + Err: err, + Object: filename, } } for _, id := range ids { - if err = n.objectDelete(ctx, delParams{addr: refs.Address{CID: cid, ObjectID: id}}); err != nil { + addr := object.NewAddress() + addr.SetObjectID(id) + addr.SetContainerID(cid) + + if err = n.objectDelete(ctx, addr); err != nil { return &api.DeleteError{ Err: err, - Object: object, + Object: filename, } } } diff --git a/api/layer/object.go b/api/layer/object.go index a8906fac..0d46ebd0 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -1,590 +1,209 @@ package layer import ( + "bytes" "context" "io" + "net/http" "time" - "github.com/nspcc-dev/neofs-api-go/object" - "github.com/nspcc-dev/neofs-api-go/query" - "github.com/nspcc-dev/neofs-api-go/refs" - "github.com/nspcc-dev/neofs-api-go/service" - "github.com/nspcc-dev/neofs-api-go/storagegroup" - "github.com/nspcc-dev/neofs-s3-gate/api/pool" + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/container" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-s3-gate/api" + "github.com/nspcc-dev/neofs-s3-gate/auth" "github.com/pkg/errors" - "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) -const ( - dataChunkSize = 3 * object.UnitsMB - objectVersion = 1 -) - type ( - putParams struct { - addr refs.Address - name string - size int64 - r io.Reader - userHeaders map[string]string - } - - sgParams struct { - addr refs.Address - objects []refs.ObjectID - } - - delParams struct { - addr refs.Address + findParams struct { + key string + cid *container.ID } getParams struct { - addr refs.Address - start int64 + io.Writer + + addr *object.Address + + offset int64 length int64 - writer io.Writer } ) -// objectSearchContainer returns all available objects in the container. -func (n *layer) objectSearchContainer(ctx context.Context, cid refs.CID) ([]refs.ObjectID, error) { - var q query.Query - q.Filters = append(q.Filters, query.Filter{ - Type: query.Filter_Exact, - Name: object.KeyRootObject, - }) - - conn, err := n.cli.GetConnection(ctx) +func (n *layer) prepareClient(ctx context.Context) (*client.Client, *token.SessionToken, error) { + conn, err := n.cli.Connection(ctx) if err != nil { - return nil, err + return nil, nil, err } - queryBinary, err := q.Marshal() + tkn, err := n.cli.Token(ctx, conn) if err != nil { - return nil, err + return nil, nil, err } - token, err := n.cli.SessionToken(ctx, &pool.SessionParams{ - Conn: conn, - Addr: refs.Address{CID: cid}, - Verb: service.Token_Info_Search, - }) + cli, err := client.New(n.key, client.WithGRPCConnection(conn)) if err != nil { - return nil, err + return nil, nil, err } - req := new(object.SearchRequest) - req.Query = queryBinary - req.QueryVersion = 1 - req.ContainerID = cid - req.SetTTL(service.SingleForwardingTTL) - req.SetToken(token) - // req.SetBearer(bearerToken) - - err = service.SignRequestData(n.key, req) - if err != nil { - return nil, err - } - - // todo: think about timeout - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - searchClient, err := object.NewServiceClient(conn).Search(ctx, req) - if err != nil { - return nil, err - } - - var ( - response []refs.Address - result []refs.ObjectID - ) - - for { - resp, err := searchClient.Recv() - if err != nil { - if err == io.EOF { - break - } - - return nil, errors.New("search command received error") - } - - response = append(response, resp.Addresses...) - } - - for i := range response { - result = append(result, response[i].ObjectID) - } - - return result, nil + return cli, tkn, nil } -// objectFindIDs returns object id's (uuid) based on they nice name in s3. If -// nice name is uuid compatible, then function returns it. -func (n *layer) objectFindIDs(ctx context.Context, cid refs.CID, name string) ([]refs.ObjectID, error) { - var q query.Query - - q.Filters = append(q.Filters, query.Filter{ - Type: query.Filter_Exact, - Name: object.KeyRootObject, - }) - q.Filters = append(q.Filters, query.Filter{ - Type: query.Filter_Exact, - Name: AWS3NameHeader, - Value: name, - }) - - queryBinary, err := q.Marshal() +// objectSearch returns all available objects by search params. +func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]*object.ID, error) { + cli, tkn, err := n.prepareClient(ctx) if err != nil { return nil, err } - conn, err := n.cli.GetConnection(ctx) - if err != nil { - return nil, err + filter := object.NewSearchFilters() + filter.AddNonLeafFilter() + + sop := new(client.SearchObjectParams) + + if p.cid != nil { + filter.AddFilter(object.HdrSysNameCID, p.cid.String(), object.MatchStringEqual) + sop.WithContainerID(p.cid) } - token, err := n.cli.SessionToken(ctx, &pool.SessionParams{ - Conn: conn, - Addr: refs.Address{CID: cid}, - Verb: service.Token_Info_Search, - }) - if err != nil { - return nil, err + if p.key != "" { + filter.AddFilter(AWS3NameHeader, p.key, object.MatchStringEqual) } - req := new(object.SearchRequest) - req.Query = queryBinary - req.QueryVersion = 1 - req.ContainerID = cid - req.SetTTL(service.SingleForwardingTTL) - req.SetToken(token) - // req.SetBearer(bearerToken) + sop.WithSearchFilters(filter) - err = service.SignRequestData(n.key, req) - if err != nil { - return nil, err - } - - // todo: think about timeout - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - searchClient, err := object.NewServiceClient(conn).Search(ctx, req) - if err != nil { - return nil, err - } - - var response []refs.Address - - for { - resp, err := searchClient.Recv() - if err != nil { - if err == io.EOF { - break - } - - return nil, errors.New("search command received error") - } - - response = append(response, resp.Addresses...) - } - - switch ln := len(response); { - case ln > 0: - result := make([]refs.ObjectID, 0, len(response)) - for i := range response { - result = append(result, response[i].ObjectID) - } - - return result, nil - default: - return nil, errors.New("object not found") - } + return cli.SearchObject(ctx, sop, client.WithSession(tkn)) } // objectFindID returns object id (uuid) based on it's nice name in s3. If // nice name is uuid compatible, then function returns it. -func (n *layer) objectFindID(ctx context.Context, cid refs.CID, name string, put bool) (refs.ObjectID, error) { - var id refs.ObjectID - - if result, err := n.objectFindIDs(ctx, cid, name); err != nil { - return id, err +func (n *layer) objectFindID(ctx context.Context, p *findParams) (*object.ID, error) { + if result, err := n.objectSearch(ctx, p); err != nil { + return nil, err } else if ln := len(result); ln == 0 { - // Minio lists all objects with and without nice names. All objects - // without nice name still have "name" in terms of minio - uuid encoded - // into string. There is a tricky case when user upload object - // with nice name that is encoded uuid. - // There is an optimisation to parse name and return uuid if it name is uuid - // compatible. It _should not_ work in case of put operation, because object - // with uuid compatible nice name may not exist. Therefore this optimization - // breaks object put logic and must be turned off. - if !put { - err := id.Parse(name) - if err == nil { - return id, nil - } - } - return id, status.Error(codes.NotFound, "object not found") + return nil, status.Error(codes.NotFound, "object not found") } else if ln == 1 { return result[0], nil } - return id, errors.New("several objects with the same name found") + return nil, errors.New("several objects with the same name found") } // objectHead returns all object's headers. -func (n *layer) objectHead(ctx context.Context, addr refs.Address) (*object.Object, error) { - - conn, err := n.cli.GetConnection(ctx) +func (n *layer) objectHead(ctx context.Context, addr *object.Address) (*object.Object, error) { + cli, tkn, err := n.prepareClient(ctx) if err != nil { return nil, err } - token, err := n.cli.SessionToken(ctx, &pool.SessionParams{ - Conn: conn, - Addr: addr, - Verb: service.Token_Info_Head, - }) - if err != nil { - return nil, err - } + ohp := new(client.ObjectHeaderParams) + ohp.WithAddress(addr) + ohp.WithAllFields() + ohp.WithMainFields() - req := new(object.HeadRequest) - req.Address = addr - req.FullHeaders = true - req.SetTTL(service.SingleForwardingTTL) - req.SetToken(token) - // req.SetBearer(bearerToken) - - err = service.SignRequestData(n.key, req) - if err != nil { - return nil, err - } - - // todo: think about timeout - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - res, err := object.NewServiceClient(conn).Head(ctx, req) - if err != nil { - return nil, err - } - - return res.Object, nil -} - -func receiveObject(cli object.Service_GetClient) (*object.Object, error) { - var ( - off int - buf []byte - obj *object.Object - ) - - for { - resp, err := cli.Recv() - if err != nil { - if err == io.EOF { - break - } - - return nil, err - } - - switch o := resp.R.(type) { - case *object.GetResponse_Object: - if obj != nil { - return nil, errors.New("object headers already received") - } else if _, hdr := o.Object.LastHeader(object.HeaderType(object.TombstoneHdr)); hdr != nil { - return nil, errors.New("object already removed") - } - - obj = o.Object - buf = make([]byte, obj.SystemHeader.PayloadLength) - - if len(obj.Payload) > 0 { - off += copy(buf, obj.Payload) - } - case *object.GetResponse_Chunk: - if obj == nil { - return nil, errors.New("object headers not received") - } - off += copy(buf[off:], o.Chunk) - default: - return nil, errors.Errorf("unknown response %T", o) - } - } - - if obj == nil { - return nil, errors.New("object headers not received") - } - obj.Payload = buf - - return obj, nil + return cli.GetObjectHeader(ctx, ohp, client.WithSession(tkn)) } // objectGet and write it into provided io.Reader. -func (n *layer) objectGet(ctx context.Context, p getParams) (*object.Object, error) { - conn, err := n.cli.GetConnection(ctx) +func (n *layer) objectGet(ctx context.Context, p *getParams) (*object.Object, error) { + cli, tkn, err := n.prepareClient(ctx) if err != nil { return nil, err } - token, err := n.cli.SessionToken(ctx, &pool.SessionParams{ - Conn: conn, - Addr: p.addr, - Verb: service.Token_Info_Get, - }) - if err != nil { - return nil, err - } + // prepare length/offset writer + writer := newWriter(p.Writer, p.offset, p.length) - // todo: replace object.Get() call by object.GetRange() for - // true sequential reading support; it will be possible when - // object.GetRange() response message become gRPC stream. - req := new(object.GetRequest) - req.Address = p.addr - req.SetTTL(service.SingleForwardingTTL) - req.SetToken(token) - // req.SetBearer(bearerToken) + gop := new(client.GetObjectParams) + gop.WithPayloadWriter(writer) - err = service.SignRequestData(n.key, req) - if err != nil { - return nil, err - } - - // todo: think about timeout - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - var obj *object.Object - - if cli, err := object.NewServiceClient(conn).Get(ctx, req); err != nil { - return nil, err - } else if obj, err = receiveObject(cli); err != nil { - return nil, err - } else if ln := int64(obj.SystemHeader.PayloadLength); p.start+p.length > ln { - return nil, errors.Errorf("slice bounds out of range: len = %d, start = %d, offset = %d", - ln, p.start, p.length) - } else if _, err = p.writer.Write(obj.Payload[p.start : p.start+p.length]); err != nil { - return nil, err - } - - // remove payload: - obj.Payload = nil - - return obj, nil + return cli.GetObject(ctx, gop, client.WithSession(tkn)) } -// objectPut into neofs, took payload from io.Reader. -func (n *layer) objectPut(ctx context.Context, p putParams) (*object.Object, error) { - conn, err := n.cli.GetConnection(ctx) - if err != nil { +// objectPut into NeoFS, took payload from io.Reader. +func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) { + var ( + err error + own *owner.ID + brt *token.BearerToken + cid = container.NewID() + ) + + if brt, err = auth.GetBearerToken(ctx); err != nil { return nil, err - } - - token, err := n.cli.SessionToken(ctx, &pool.SessionParams{ - Conn: conn, - Addr: p.addr, - Verb: service.Token_Info_Put, - }) - if err != nil { - n.log.Error("could not prepare token", - zap.Error(err)) + } else if own, err = GetOwnerID(brt); err != nil { return nil, err - } - - putClient, err := object.NewServiceClient(conn).Put(ctx) - if err != nil { - n.log.Error("could not prepare PutClient", - zap.Error(err)) + } else if err = cid.Parse(p.Bucket); err != nil { return nil, err - } - - if p.userHeaders == nil { - p.userHeaders = make(map[string]string) - } - - // Set object name if not set before - if _, ok := p.userHeaders[AWS3NameHeader]; !ok { - p.userHeaders[AWS3NameHeader] = p.name - } - - readBuffer := make([]byte, dataChunkSize) - obj := &object.Object{ - SystemHeader: object.SystemHeader{ - Version: objectVersion, - ID: p.addr.ObjectID, - OwnerID: n.uid, - CID: p.addr.CID, - PayloadLength: uint64(p.size), - }, - Headers: parseUserHeaders(p.userHeaders), - } - - req := object.MakePutRequestHeader(obj) - req.SetTTL(service.SingleForwardingTTL) - req.SetToken(token) - // req.SetBearer(bearerToken) - - err = service.SignRequestData(n.key, req) - if err != nil { - n.log.Error("could not prepare request", - zap.Error(err)) - return nil, err - } - - err = putClient.Send(req) - if err != nil { - n.log.Error("could not send request", - zap.Error(err)) - return nil, err - } - - read, err := p.r.Read(readBuffer) - for read > 0 { - if err != nil && err != io.EOF { - n.log.Error("something went wrong", - zap.Error(err)) - return nil, err + } else if _, err = n.objectFindID(ctx, &findParams{cid: cid, key: p.Object}); err == nil { + return nil, &api.ObjectAlreadyExists{ + Bucket: p.Bucket, + Object: p.Object, } - - if read > 0 { - req := object.MakePutRequestChunk(readBuffer[:read]) - req.SetTTL(service.SingleForwardingTTL) - // req.SetBearer(bearerToken) - - err = service.SignRequestData(n.key, req) - if err != nil { - n.log.Error("could not sign chunk request", - zap.Error(err)) - return nil, err - } - - err = putClient.Send(req) - if err != nil && err != io.EOF { - n.log.Error("could not send chunk", - zap.Error(err)) - return nil, err - } - } - - read, err = p.r.Read(readBuffer) } - _, err = putClient.CloseAndRecv() - if err != nil { - n.log.Error("could not finish request", - zap.Error(err)) - return nil, err - } - - // maybe make a head? - return obj, nil -} - -// storageGroupPut prepares storage group object and put it into neofs. -func (n *layer) storageGroupPut(ctx context.Context, p sgParams) (*object.Object, error) { - conn, err := n.cli.GetConnection(ctx) + cli, tkn, err := n.prepareClient(ctx) if err != nil { return nil, err } - token, err := n.cli.SessionToken(ctx, &pool.SessionParams{ - Conn: conn, - Addr: p.addr, - Verb: service.Token_Info_Put, - }) - if err != nil { + attributes := make([]*object.Attribute, 0, len(p.Header)+1) + + filename := object.NewAttribute() + filename.SetKey(AWS3NameHeader) + filename.SetValue(p.Object) + + attributes = append(attributes, filename) + + for k, v := range p.Header { + ua := object.NewAttribute() + ua.SetKey(k) + ua.SetValue(v) + + attributes = append(attributes, ua) + } + + b := new(bytes.Buffer) + r := io.TeeReader(p.Reader, b) + + raw := object.NewRaw() + raw.SetOwnerID(own) + raw.SetContainerID(cid) + raw.SetAttributes(attributes...) + + pop := new(client.PutObjectParams) + pop.WithPayloadReader(r) + pop.WithObject(raw.Object()) + + if _, err = cli.PutObject(ctx, pop, client.WithSession(tkn)); err != nil { return nil, err } - // todo: think about timeout - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - putClient, err := object.NewServiceClient(conn).Put(ctx) - if err != nil { - return nil, err - } - - sg := &object.Object{ - SystemHeader: object.SystemHeader{ - Version: objectVersion, - ID: p.addr.ObjectID, - OwnerID: n.uid, - CID: p.addr.CID, - }, - Headers: make([]object.Header, 0, len(p.objects)), - } - - for i := range p.objects { - sg.AddHeader(&object.Header{Value: &object.Header_Link{ - Link: &object.Link{Type: object.Link_StorageGroup, ID: p.objects[i]}, - }}) - } - - sg.SetStorageGroup(new(storagegroup.StorageGroup)) - - req := object.MakePutRequestHeader(sg) - req.SetTTL(service.SingleForwardingTTL) - req.SetToken(token) - // req.SetBearer(bearerToken) - - err = service.SignRequestData(n.key, req) - if err != nil { - return nil, err - } - - err = putClient.Send(req) - if err != nil { - return nil, err - } - - _, err = putClient.CloseAndRecv() - if err != nil { - return nil, err - } - - return sg, nil + return &ObjectInfo{ + Bucket: p.Bucket, + Name: p.Object, + Size: p.Size, + Created: time.Now(), + ContentType: http.DetectContentType(b.Bytes()), + Owner: own, + Headers: p.Header, + }, nil } // objectDelete puts tombstone object into neofs. -func (n *layer) objectDelete(ctx context.Context, p delParams) error { - conn, err := n.cli.GetConnection(ctx) +func (n *layer) objectDelete(ctx context.Context, address *object.Address) error { + cli, tkn, err := n.prepareClient(ctx) if err != nil { return err } - token, err := n.cli.SessionToken(ctx, &pool.SessionParams{ - Conn: conn, - Addr: p.addr, - Verb: service.Token_Info_Delete, - }) - if err != nil { - return err - } + dob := new(client.DeleteObjectParams) + dob.WithAddress(address) - req := new(object.DeleteRequest) - req.Address = p.addr - req.OwnerID = n.uid - req.SetTTL(service.SingleForwardingTTL) - req.SetToken(token) - // req.SetBearer(bearerToken) - - err = service.SignRequestData(n.key, req) - if err != nil { - return err - } - - // todo: think about timeout - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - _, err = object.NewServiceClient(conn).Delete(ctx, req) - - return err + return cli.DeleteObject(ctx, dob, client.WithSession(tkn)) } diff --git a/api/layer/util.go b/api/layer/util.go index d97f13ed..e504e83b 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -6,8 +6,8 @@ import ( "strings" "time" - "github.com/nspcc-dev/neofs-api-go/object" - "github.com/nspcc-dev/neofs-api-go/refs" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" ) type ( @@ -17,7 +17,7 @@ type ( Size int64 ContentType string Created time.Time - Owner refs.OwnerID + Owner *owner.ID Headers map[string]string } @@ -39,7 +39,7 @@ type ( NextContinuationToken string // List of objects info for this request. - Objects []ObjectInfo + Objects []*ObjectInfo // List of prefixes for this request. Prefixes []string @@ -48,65 +48,46 @@ type ( const pathSeparator = string(os.PathSeparator) -func userHeaders(h []object.Header) map[string]string { - result := make(map[string]string, len(h)) +func userHeaders(attrs []*object.Attribute) map[string]string { + result := make(map[string]string, len(attrs)) - for i := range h { - switch v := h[i].Value.(type) { - case *object.Header_UserHeader: - result[v.UserHeader.Key] = v.UserHeader.Value - default: - continue - } + for _, attr := range attrs { + result[attr.GetKey()] = attr.GetValue() } return result } func objectInfoFromMeta(meta *object.Object) *ObjectInfo { - aws3name := meta.SystemHeader.ID.String() + aws3name := meta.GetID().String() - userHeaders := userHeaders(meta.Headers) + userHeaders := userHeaders(meta.GetAttributes()) if name, ok := userHeaders[AWS3NameHeader]; ok { aws3name = name delete(userHeaders, name) } - mimeType := http.DetectContentType(meta.Payload) + mimeType := http.DetectContentType(meta.GetPayload()) return &ObjectInfo{ - Bucket: meta.SystemHeader.CID.String(), + Bucket: meta.GetContainerID().String(), Name: aws3name, ContentType: mimeType, Headers: userHeaders, - Size: int64(meta.SystemHeader.PayloadLength), - Created: time.Unix(meta.SystemHeader.CreatedAt.UnixTime, 0), + Size: int64(meta.GetPayloadSize()), + Created: time.Now(), // time.Unix(meta.GetCreationEpoch(), 0), } } -func parseUserHeaders(h map[string]string) []object.Header { - headers := make([]object.Header, 0, len(h)) - - for k, v := range h { - uh := &object.UserHeader{Key: k, Value: v} - headers = append(headers, object.Header{ - Value: &object.Header_UserHeader{UserHeader: uh}, - }) - } - - return headers -} - func nameFromObject(o *object.Object) (string, string) { - var ( - name string - uh = userHeaders(o.Headers) - ) + var name = o.GetID().String() - if _, ok := uh[AWS3NameHeader]; !ok { - name = o.SystemHeader.ID.String() - } else { - name = uh[AWS3NameHeader] + for _, attr := range o.GetAttributes() { + if attr.GetKey() == AWS3NameHeader { + name = attr.GetValue() + + break + } } ind := strings.LastIndex(name, pathSeparator) diff --git a/api/layer/writer.go b/api/layer/writer.go new file mode 100644 index 00000000..98f24b89 --- /dev/null +++ b/api/layer/writer.go @@ -0,0 +1,49 @@ +package layer + +import "io" + +type offsetWriter struct { + io.Writer + + written int64 + skipped int64 + + offset int64 + length int64 +} + +func newWriter(w io.Writer, offset, length int64) io.Writer { + return &offsetWriter{ + Writer: w, + offset: offset, + length: length, + } +} + +func (w *offsetWriter) Write(p []byte) (int, error) { + ln := len(p) + length := int64(ln) + offset := w.offset - w.skipped + + if length-offset < 0 { + w.skipped += length + + return ln, nil + } + + length -= offset + + left := w.length - w.written + if left-length < 0 || length-left < length { + length = left + } else { + return 0, nil + } + + n, err := w.Writer.Write(p[offset : offset+length]) + + w.written += int64(n) + w.skipped += offset + + return n, err +} diff --git a/api/layer/writer_test.go b/api/layer/writer_test.go new file mode 100644 index 00000000..9c820f0c --- /dev/null +++ b/api/layer/writer_test.go @@ -0,0 +1,77 @@ +package layer + +import ( + "bytes" + "crypto/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func testBuffer(t *testing.T) []byte { + buf := make([]byte, 1024) + _, err := rand.Read(buf) + require.NoError(t, err) + + return buf +} + +func TestOffsetWriter(t *testing.T) { + b := testBuffer(t) + k := 64 + d := len(b) / k + + t.Run("1024 / 100 / 100 bytes success", func(t *testing.T) { + w := new(bytes.Buffer) + o := int64(100) + l := int64(100) + + wt := newWriter(w, o, l) + for i := 0; i < k; i++ { + _, err := wt.Write(b[i*d : (i+1)*d]) + require.NoError(t, err) + } + + wo := wt.(*offsetWriter) + + require.Equal(t, o, wo.skipped) + require.Equal(t, l, wo.written) + require.Equal(t, b[o:o+l], w.Bytes()) + }) + + t.Run("1024 / 0 / 100 bytes success", func(t *testing.T) { + w := new(bytes.Buffer) + o := int64(0) + l := int64(100) + + wt := newWriter(w, o, l) + for i := 0; i < k; i++ { + _, err := wt.Write(b[i*d : (i+1)*d]) + require.NoError(t, err) + } + + wo := wt.(*offsetWriter) + + require.Equal(t, o, wo.skipped) + require.Equal(t, l, wo.written) + require.Equal(t, b[o:o+l], w.Bytes()) + }) + + t.Run("1024 / 0 / 1024 bytes success", func(t *testing.T) { + w := new(bytes.Buffer) + o := int64(0) + l := int64(1024) + + wt := newWriter(w, o, l) + for i := 0; i < k; i++ { + _, err := wt.Write(b[i*d : (i+1)*d]) + require.NoError(t, err) + } + + wo := wt.(*offsetWriter) + + require.Equal(t, o, wo.skipped) + require.Equal(t, l, wo.written) + require.Equal(t, b[o:o+l], w.Bytes()) + }) +} From b72a957341d016e713868c5e59d264ed37bab7a3 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 19 Oct 2020 04:05:08 +0300 Subject: [PATCH 08/13] [#25] Migrate handler to NeoFS API v2 Naive migration, without any tries to start application closes #25 Signed-off-by: Evgeniy Kulikov --- api/handler/list.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/api/handler/list.go b/api/handler/list.go index 10c983f5..2649c974 100644 --- a/api/handler/list.go +++ b/api/handler/list.go @@ -5,6 +5,8 @@ import ( "strconv" "time" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-s3-gate/api" "github.com/nspcc-dev/neofs-s3-gate/api/layer" "github.com/nspcc-dev/neofs-s3-gate/auth" @@ -25,12 +27,14 @@ var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/li func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { var ( + err error + own = owner.NewID() + tkn *token.BearerToken res *ListBucketsResponse rid = api.GetRequestID(r.Context()) ) - tkn, err := auth.GetBearerToken(r.Context()) - if err != nil { + if tkn, err = auth.GetBearerToken(r.Context()); err != nil { h.log.Error("something went wrong", zap.String("request_id", rid), zap.Error(err)) @@ -41,6 +45,18 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { HTTPStatusCode: http.StatusInternalServerError, }, r.URL) + return + } else if own, err = layer.GetOwnerID(tkn); err != nil { + h.log.Error("something went wrong", + zap.String("request_id", rid), + zap.Error(err)) + + api.WriteErrorResponse(r.Context(), w, api.Error{ + Code: api.GetAPIError(api.ErrBadRequest).Code, + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + }, r.URL) + return } @@ -61,8 +77,8 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { res = &ListBucketsResponse{ Owner: Owner{ - ID: tkn.OwnerID.String(), - DisplayName: tkn.OwnerID.String(), + ID: own.String(), + DisplayName: own.String(), }, } From 424fb7a1d1d9d9a35d8ed4ee14a0fbffa87fe10d Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 19 Oct 2020 04:05:28 +0300 Subject: [PATCH 09/13] [#25] Migrate cmd/gate to NeoFS API v2 Naive migration, without any tries to start application closes #25 Signed-off-by: Evgeniy Kulikov --- cmd/gate/app.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cmd/gate/app.go b/cmd/gate/app.go index baf2cddf..de24a7c3 100644 --- a/cmd/gate/app.go +++ b/cmd/gate/app.go @@ -3,8 +3,10 @@ package main import ( "context" "crypto/ecdsa" + "errors" "net" "net/http" + "os" "time" "github.com/nspcc-dev/neofs-authmate/accessbox/hcs" @@ -148,12 +150,24 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App { cli.ReBalance(ctx) if _, err = cli.Connection(ctx); err != nil { + if errors.Is(err, context.Canceled) { + l.Info("connection canceled") + os.Exit(0) + } + l.Fatal("could not establish connection", zap.Error(err)) } } - if obj, err = layer.NewLayer(l, cli, nfKey); err != nil { + layerParams := &layer.Params{ + Pool: cli, + Logger: l, + Timeout: reqTimeout, + NFKey: nfKey, + } + + if obj, err = layer.NewLayer(layerParams); err != nil { l.Fatal("could not prepare ObjectLayer", zap.Error(err)) } From 14517d682cf46f0823d02c7817d71cc31df31cd0 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 19 Oct 2020 11:47:13 +0300 Subject: [PATCH 10/13] [#25] Show default environments - Add possibility to display default environments - Add Prefix constant - Update README closes #25 Signed-off-by: Evgeniy Kulikov --- README.md | 52 +++++++++++++++++++++++++++++-- cmd/gate/app-settings.go | 66 ++++++++++++++++++++++++++++++++-------- misc/build.go | 6 +++- 3 files changed, 109 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 52eac484..00458113 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,59 @@ # NeoFS S3 Gate -... +## Example of configuration + +``` +# Flags + --pprof enable pprof + --metrics enable prometheus metrics + -h, --help show help + -v, --version show version + --neofs-key string set value to hex string, WIF string, or path to NeoFS private key file (use "generated" to generate key) (default "generated") + --auth-key string set path to file with auth (curve25519) private key to use in auth scheme + --verbose set debug mode of gRPC connections + --request_timeout duration set gRPC request timeout (default 15s) + --connect_timeout duration set gRPC connect timeout (default 30s) + --rebalance_timer duration set gRPC connection rebalance timer (default 15s) + --max_clients_count int set max-clients count (default 100) + --max_clients_deadline duration set max-clients deadline (default 30s) + -t, --con_ttl duration set gRPC connection time to live (default 5m0s) + --listen_address string set address to listen (default "0.0.0.0:8080") + -p, --peers stringArray set NeoFS nodes + +# Environments + +S3_AUTH-KEY = +S3_CON_TTL = 5m0s +S3_CONNECT_TIMEOUT = 30s +S3_KEEPALIVE_PERMIT_WITHOUT_STREAM = true +S3_KEEPALIVE_TIME = 10s +S3_KEEPALIVE_TIMEOUT = 10s +S3_LISTEN_ADDRESS = 0.0.0.0:8080 +S3_LOGGER_FORMAT = console +S3_LOGGER_LEVEL = debug +S3_LOGGER_NO_DISCLAIMER = true +S3_LOGGER_SAMPLING_INITIAL = 1000 +S3_LOGGER_SAMPLING_THEREAFTER = 1000 +S3_LOGGER_TRACE_LEVEL = panic +S3_MAX_CLIENTS_COUNT = 100 +S3_MAX_CLIENTS_DEADLINE = 30s +S3_METRICS = false +S3_NEOFS-KEY = generated +S3_PPROF = false +S3_REBALANCE_TIMER = 15s +S3_REQUEST_TIMEOUT = 15s +S3_VERBOSE = false + +# Peers preset + +S3_PEERS_[N]_ADDRESS = string +S3_PEERS_[N]_WEIGHT = 0..1 (float) +``` ---