From e0b87852d519cf31cee961e9c07f7af7ace9dd4c Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Thu, 1 Jan 1970 00:00:00 +0000 Subject: [PATCH 001/548] Empty default branch for forked repos Signed-off-by: Vitaliy Potyarkin --- README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..7463f9e --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# WIP area: this repo is just a fork! + +Useful things may be published only in [other branches](../../../branches) From b4ac08d341e5e9c298433fd4cb2802ffb338bdea Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 6 Nov 2019 15:33:46 +0300 Subject: [PATCH 002/548] initial --- .gitignore | 8 ++ Dockerfile | 23 +++++ Makefile | 29 ++++++ go.mod | 20 +++++ go.sum | 260 +++++++++++++++++++++++++++++++++++++++++++++++++++++ grace.go | 24 +++++ logger.go | 89 ++++++++++++++++++ main.go | 260 +++++++++++++++++++++++++++++++++++++++++++++++++++++ misc.go | 10 +++ 9 files changed, 723 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 go.mod create mode 100644 go.sum create mode 100644 grace.go create mode 100644 logger.go create mode 100644 main.go create mode 100644 misc.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..690f5d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +bin +temp +cmd/test +/plugins/ +/vendor/ + +testfile +.neofs-cli.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9e28b96 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1-alpine as builder + +ARG BUILD=now +ARG VERSION=dev +ARG REPO=github.com/nspcc-dev/neofs-gw + +ENV GOGC off +ENV CGO_ENABLED 0 +ENV LDFLAGS "-w -s -X ${REPO}/Version=${VERSION} -X ${REPO}/Build=${BUILD}" + +WORKDIR /src + +COPY . /src + +RUN go build -v -mod=vendor -ldflags "${LDFLAGS}" -o /go/bin/neofs-gw ./ + +# Executable image +FROM scratch + +WORKDIR / + +COPY --from=builder /go/bin/neofs-gw /bin/neofs-gw +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8729d78 --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +REPO ?= $(shell go list -m) +VERSION ?= "$(shell git describe --tags 2>/dev/null | sed 's/^v//')" + +HUB_IMAGE=nspccdev/neofs + +B=\033[0;1m +G=\033[0;92m +R=\033[0m + +# Show current version +version: + @echo $(VERSION) + +# Make sure that all files added to commit +vendor: + @printf "${B}${G}⇒ Ensure vendor${R}: " + @GOPRIVATE=bitbucket.org/nspcc-dev go mod tidy -v && echo OK || (echo fail && exit 2) + @printf "${B}${G}⇒ Download requirements${R}: " + @GOPRIVATE=bitbucket.org/nspcc-dev go mod download && echo OK || (echo fail && exit 2) + @printf "${B}${G}⇒ Store vendor localy${R}: " + @GOPRIVATE=bitbucket.org/nspcc-dev go mod vendor && echo OK || (echo fail && exit 2) + +image: vendor + @echo "${B}${G}⇒ Build GW docker-image ${R}" + @docker build \ + --build-arg REPO=$(REPO) \ + --build-arg VERSION=$(VERSION) \ + -f Dockerfile \ + -t $(HUB_IMAGE)-http-gate:$(VERSION) . diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..12b0157 --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module github.com/nspcc-dev/neofs-gw + +go 1.13 + +require ( + github.com/labstack/echo/v4 v4.1.11 + github.com/nspcc-dev/hrw v1.0.9 // indirect + github.com/nspcc-dev/neofs-proto v0.0.0-20191101093315-0d5a92f60568 + github.com/pkg/errors v0.8.1 + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.5.0 + go.uber.org/zap v1.11.0 + golang.org/x/text v0.3.2 // indirect + google.golang.org/grpc v1.24.0 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + +) + +// Temporary, before we move repo to github: +replace github.com/nspcc-dev/neofs-proto v0.0.0-20191101093315-0d5a92f60568 => bitbucket.org/nspcc-dev/neofs-proto v0.0.0-20191101093315-0d5a92f60568 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7d2e17d --- /dev/null +++ b/go.sum @@ -0,0 +1,260 @@ +bitbucket.org/nspcc-dev/neofs-proto v0.0.0-20191101093315-0d5a92f60568 h1:Uy8dzzGOqK0QPJauQI2HseW2XwUIDwUq52mJjyrQqb8= +bitbucket.org/nspcc-dev/neofs-proto v0.0.0-20191101093315-0d5a92f60568/go.mod h1:SfhT/MhBkpJVCVS+hK/APVwHKtiEAnN9bNxEFJ2kx+E= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c h1:2RuXx1+tSNWRjxhY0Bx52kjV2odJQ0a6MTbfTPhGAkg= +code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= +github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310 h1:t+qxRrRtwNiUYA+Xh2jSXhoG2grnMCMKX4Fg6lx9X1U= +github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/labstack/echo/v4 v4.1.11 h1:z0BZoArY4FqdpUEl+wlHp4hnr/oSR6MTmQmv8OHSoww= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= +github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= +github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= +github.com/nspcc-dev/neofs-crypto v0.1.0 h1:G5Nz5AO2ALwRCFgQkgtifZZPfPqZtGd9/qu6OoEF9EA= +github.com/nspcc-dev/neofs-crypto v0.1.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= +github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= +github.com/nspcc-dev/netmap v1.6.1/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= +github.com/nspcc-dev/rfc6979 v0.1.0 h1:Lwg7esRRoyK1Up/IN1vAef1EmvrBeMHeeEkek2fAJ6c= +github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= +github.com/nspcc-dev/tzhash v1.3.0 h1:n6FTHsfPYbMi5Jmo6SwGVVRQD8i2w1P2ScCaW6rz69Q= +github.com/nspcc-dev/tzhash v1.3.0/go.mod h1:Lc4DersKS8MNIrunTmsAzANO56qnG+LZ4GOE/WYGVzU= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.5.0 h1:GpsTwfsQ27oS/Aha/6d1oD7tpKIqWnOA6tgOX9HHkt4= +github.com/spf13/viper v1.5.0/go.mod h1:AkYRkVJF8TkSG/xet6PzXX+l39KhhXa2pdqVSxnTcn4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.11.0 h1:gSmpCfs+R47a4yQPAI4xJ0IPDLTRGXskm6UelqNXpqE= +go.uber.org/zap v1.11.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/grace.go b/grace.go new file mode 100644 index 0000000..8469d62 --- /dev/null +++ b/grace.go @@ -0,0 +1,24 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + + "go.uber.org/zap" +) + +// newGracefulContext returns graceful context +func newGracefulContext(l *zap.Logger) context.Context { + ctx, cancel := context.WithCancel(context.Background()) + go func() { + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + sig := <-ch + l.Info("received signal", + zap.String("signal", sig.String())) + cancel() + }() + return ctx +} diff --git a/logger.go b/logger.go new file mode 100644 index 0000000..07fa9b1 --- /dev/null +++ b/logger.go @@ -0,0 +1,89 @@ +package main + +import ( + "strings" + + "github.com/spf13/viper" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +const ( + formatJSON = "json" + formatConsole = "console" + + defaultSamplingInitial = 100 + defaultSamplingThereafter = 100 +) + +func safeLevel(lvl string) zap.AtomicLevel { + switch strings.ToLower(lvl) { + case "debug": + return zap.NewAtomicLevelAt(zap.DebugLevel) + case "warn": + return zap.NewAtomicLevelAt(zap.WarnLevel) + case "error": + return zap.NewAtomicLevelAt(zap.ErrorLevel) + case "fatal": + return zap.NewAtomicLevelAt(zap.FatalLevel) + case "panic": + return zap.NewAtomicLevelAt(zap.PanicLevel) + default: + return zap.NewAtomicLevelAt(zap.InfoLevel) + } +} + +func newLogger(v *viper.Viper) (*zap.Logger, error) { + c := zap.NewProductionConfig() + + c.OutputPaths = []string{"stdout"} + c.ErrorOutputPaths = []string{"stdout"} + + if v.IsSet("logger.sampling") { + c.Sampling = &zap.SamplingConfig{ + Initial: defaultSamplingInitial, + Thereafter: defaultSamplingThereafter, + } + + if val := v.GetInt("logger.sampling.initial"); val > 0 { + c.Sampling.Initial = val + } + + if val := v.GetInt("logger.sampling.thereafter"); val > 0 { + c.Sampling.Thereafter = val + } + } + + // logger level + c.Level = safeLevel(v.GetString("logger.level")) + traceLvl := safeLevel(v.GetString("logger.trace_level")) + + // logger format + switch f := v.GetString("logger.format"); strings.ToLower(f) { + case formatConsole: + c.Encoding = formatConsole + default: + c.Encoding = formatJSON + } + + // logger time + c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + + l, err := c.Build( + // enable trace only for current log-level + zap.AddStacktrace(traceLvl)) + if err != nil { + return nil, err + } + + if v.GetBool("logger.no_disclaimer") { + return l, nil + } + + name := v.GetString("app.name") + version := v.GetString("app.version") + + return l.With( + zap.String("app_name", name), + zap.String("app_version", version)), nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..77c4dc3 --- /dev/null +++ b/main.go @@ -0,0 +1,260 @@ +package main + +import ( + "context" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/labstack/echo/v4" + "github.com/nspcc-dev/neofs-proto/object" + "github.com/nspcc-dev/neofs-proto/refs" + "github.com/nspcc-dev/neofs-proto/service" + "github.com/pkg/errors" + "github.com/spf13/viper" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/keepalive" +) + +type config struct { + log *zap.Logger + timeout time.Duration + cli object.ServiceClient +} + +var defaultConfig = strings.NewReader(` +request_timeout: 5s +connect_timeout: 30s +listen_address: :8082 +neofs_node_addr: :8080 + +logger: + level: debug + format: console + trace_level: fatal + no_disclaimer: true + sampling: + initial: 1000 + thereafter: 1000 + +keepalive: + time: 100ms + timeout: 10s + permit_without_stream: true +`) + +func main() { + v := viper.New() + v.AutomaticEnv() + v.SetEnvPrefix("GW") + v.SetConfigType("yaml") + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + + // set defaults: + v.Set("app.name", "neofs-gw") + v.Set("app.version", Version) + + if err := v.ReadConfig(defaultConfig); err != nil { + panic(err) + } + + log, err := newLogger(v) + if err != nil { + panic(err) + } + + log.Info("running application", zap.String("version", v.GetString("app.version"))) + + var ( + cfg = new(config) + grace = newGracefulContext(log) + ) + + cfg.log = log + cfg.timeout = v.GetDuration("request_timeout") + ctx, cancel := context.WithTimeout(grace, v.GetDuration("connect_timeout")) + defer cancel() + + conn, err := grpc.DialContext(ctx, v.GetString("neofs_node_addr"), + grpc.WithInsecure(), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: v.GetDuration("keepalive.time"), + Timeout: v.GetDuration("keepalive.timeout"), + PermitWithoutStream: v.GetBool("keepalive.permit_without_stream"), + }), + ) + if err != nil { + log.Panic("could not connect to neofs-node", + zap.String("neofs-node", v.GetString("neofs_node_addr")), + zap.Error(err)) + } + + ctx, cancel = context.WithCancel(grace) + defer cancel() + + go checkConnection(ctx, conn, log) + cfg.cli = object.NewServiceClient(conn) + + e := echo.New() + e.Debug = false + e.HidePort = true + e.HideBanner = true + + e.GET("/:cid/:oid", cfg.receiveFile) + go func() { + log.Info("run gateway server", + zap.String("address", v.GetString("listen_address"))) + if err := e.Start(v.GetString("listen_address")); err != nil { + log.Panic("could not start server", zap.Error(err)) + } + }() + + <-ctx.Done() + + ctx, cancel = context.WithTimeout(context.TODO(), time.Second*30) + defer cancel() + + if err := e.Shutdown(ctx); err != nil { + log.Panic("could not stop server", zap.Error(err)) + } +} + +func checkConnection(ctx context.Context, conn *grpc.ClientConn, log *zap.Logger) { + tick := time.NewTicker(time.Second) +loop: + for { + select { + case <-ctx.Done(): + break loop + case <-tick.C: + switch state := conn.GetState(); state { + case connectivity.Idle, connectivity.Connecting, connectivity.Ready: + // It's ok.. + default: + log.Panic("could not establish connection", + zap.Stringer("state", state), + zap.Any("connection", conn.Target())) + } + } + } + + tick.Stop() +} + +func (cfg *config) receiveFile(c echo.Context) error { + var ( + cid refs.CID + oid refs.ObjectID + obj *object.Object + download = c.QueryParam("download") != "" + ) + + cfg.log.Debug("try to fetch object from network", + zap.String("cid", c.Param("cid")), + zap.String("oid", c.Param("oid"))) + + if err := cid.Parse(c.Param("cid")); err != nil { + cfg.log.Error("wrong container id", + zap.String("cid", c.Param("cid")), + zap.String("oid", c.Param("oid")), + zap.Error(err)) + + return echo.NewHTTPError( + http.StatusBadRequest, + errors.Wrap(err, "wrong container id").Error(), + ) + } else if err := oid.Parse(c.Param("oid")); err != nil { + cfg.log.Error("wrong object id", + zap.Stringer("cid", cid), + zap.String("oid", c.Param("oid")), + zap.Error(err)) + + return echo.NewHTTPError( + http.StatusBadRequest, + errors.Wrap(err, "wrong object id").Error(), + ) + } + + ctx, cancel := context.WithTimeout(context.Background(), cfg.timeout) + defer cancel() + + cli, err := cfg.cli.Get(ctx, &object.GetRequest{ + Address: refs.Address{ObjectID: oid, CID: cid}, + TTL: service.SingleForwardingTTL, + }) + + if err != nil { + cfg.log.Error("could not prepare connection", + zap.Stringer("cid", cid), + zap.Stringer("oid", oid), + zap.Error(err)) + + return echo.NewHTTPError( + // TODO: nginx doesn't return 500 errors from backend + // http.StatusInternalServerError, + http.StatusBadRequest, + errors.Wrap(err, "could not prepare connection").Error(), + ) + } else if obj, err = receiveObject(cli); err != nil { + cfg.log.Error("could not receive object", + zap.Stringer("cid", cid), + zap.Stringer("oid", oid), + zap.Error(err)) + + if strings.Contains(err.Error(), object.ErrNotFound.Error()) { + return echo.NewHTTPError(http.StatusNotFound, err.Error()) + } + + return echo.NewHTTPError( + // TODO: nginx doesn't return 500 errors from backend + // http.StatusInternalServerError, + http.StatusBadRequest, + errors.Wrap(err, "could not receive object").Error(), + ) + } + + cfg.log.Info("object fetched successfully", + zap.Stringer("cid", cid), + zap.Stringer("oid", oid)) + + c.Response().Header().Set("Content-Length", strconv.FormatUint(obj.SystemHeader.PayloadLength, 10)) + c.Response().Header().Set("x-object-id", obj.SystemHeader.ID.String()) + c.Response().Header().Set("x-owner-id", obj.SystemHeader.OwnerID.String()) + c.Response().Header().Set("x-container-id", obj.SystemHeader.CID.String()) + + for i := range obj.Headers { + if hdr := obj.Headers[i].GetUserHeader(); hdr != nil { + c.Response().Header().Set("x-"+hdr.Key, hdr.Value) + + if hdr.Key == DropInFilenameHeader && download { + c.Response().Header().Set("Content-Disposition", "attachment; filename="+hdr.Value) + } + } + } + + return c.Blob(http.StatusOK, + http.DetectContentType(obj.Payload), + obj.Payload) +} + +func receiveObject(cli object.Service_GetClient) (*object.Object, error) { + var obj *object.Object + for { + resp, err := cli.Recv() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } else if obj == nil { + obj = resp.GetObject() + } + + obj.Payload = append(obj.Payload, resp.GetChunk()...) + } + return obj, nil +} diff --git a/misc.go b/misc.go new file mode 100644 index 0000000..48700df --- /dev/null +++ b/misc.go @@ -0,0 +1,10 @@ +package main + +const DropInFilenameHeader = "drop-in-filename" + +var ( + Prefix = "neofs" + Build = "now" + Version = "dev" + Debug = "true" +) From 99ed41b90548f6cc16d9377043809a6b794b1108 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 7 Nov 2019 20:29:29 +0300 Subject: [PATCH 003/548] NSPCC-587 Add License and Readme --- LICENSE.md | 675 +++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 25 ++ misc.go | 7 +- 3 files changed, 701 insertions(+), 6 deletions(-) create mode 100644 LICENSE.md create mode 100644 README.md diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..4d393d1 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,675 @@ +### GNU GENERAL PUBLIC LICENSE + +Version 3, 29 June 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. + + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +### Preamble + +The GNU General Public License is a free, copyleft license for +software and other kinds of works. + +The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom +to share and change all versions of a program--to make sure it remains +free software for all its users. We, the Free Software Foundation, use +the GNU General Public License for most of our software; it applies +also to any other work released this way by its authors. You can apply +it to your programs, too. + +When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you +have certain responsibilities if you distribute copies of the +software, or if you modify it: responsibilities to respect the freedom +of others. + +For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + +Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + +Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the +manufacturer can do so. This is fundamentally incompatible with the +aim of protecting users' freedom to change the software. The +systematic pattern of such abuse occurs in the area of products for +individuals to use, which is precisely where it is most unacceptable. +Therefore, we have designed this version of the GPL to prohibit the +practice for those products. If such problems arise substantially in +other domains, we stand ready to extend this provision to those +domains in future versions of the GPL, as needed to protect the +freedom of users. + +Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish +to avoid the special danger that patents applied to a free program +could make it effectively proprietary. To prevent this, the GPL +assures that patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and +modification follow. + +### TERMS AND CONDITIONS + +#### 0. Definitions. + +"This License" refers to version 3 of the GNU General Public License. + +"Copyright" also means copyright-like laws that apply to other kinds +of works, such as semiconductor masks. + +"The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + +To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of +an exact copy. The resulting work is called a "modified version" of +the earlier work or a work "based on" the earlier work. + +A "covered work" means either the unmodified Program or a work based +on the Program. + +To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + +To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user +through a computer network, with no transfer of a copy, is not +conveying. + +An interactive user interface displays "Appropriate Legal Notices" to +the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +#### 1. Source Code. + +The "source code" for a work means the preferred form of the work for +making modifications to it. "Object code" means any non-source form of +a work. + +A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + +The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + +The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can +regenerate automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same +work. + +#### 2. Basic Permissions. + +All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, +without conditions so long as your license otherwise remains in force. +You may convey covered works to others for the sole purpose of having +them make modifications exclusively for you, or provide you with +facilities for running those works, provided that you comply with the +terms of this License in conveying all material for which you do not +control copyright. Those thus making or running the covered works for +you must do so exclusively on your behalf, under your direction and +control, on terms that prohibit them from making any copies of your +copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under the +conditions stated below. Sublicensing is not allowed; section 10 makes +it unnecessary. + +#### 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + +No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + +When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such +circumvention is effected by exercising rights under this License with +respect to the covered work, and you disclaim any intention to limit +operation or modification of the work as a means of enforcing, against +the work's users, your or third parties' legal rights to forbid +circumvention of technological measures. + +#### 4. Conveying Verbatim Copies. + +You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + +#### 5. Conveying Modified Source Versions. + +You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these +conditions: + +- a) The work must carry prominent notices stating that you modified +it, and giving a relevant date. +- b) The work must carry prominent notices stating that it is +released under this License and any conditions added under +section 7. This requirement modifies the requirement in section 4 +to "keep intact all notices". +- c) You must license the entire work, as a whole, under this +License to anyone who comes into possession of a copy. This +License will therefore apply, along with any applicable section 7 +additional terms, to the whole of the work, and all its parts, +regardless of how they are packaged. This License gives no +permission to license the work in any other way, but it does not +invalidate such permission if you have separately received it. +- d) If the work has interactive user interfaces, each must display +Appropriate Legal Notices; however, if the Program has interactive +interfaces that do not display Appropriate Legal Notices, your +work need not make them do so. + +A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + +#### 6. Conveying Non-Source Forms. + +You may convey a covered work in object code form under the terms of +sections 4 and 5, provided that you also convey the machine-readable +Corresponding Source under the terms of this License, in one of these +ways: + +- a) Convey the object code in, or embodied in, a physical product +(including a physical distribution medium), accompanied by the +Corresponding Source fixed on a durable physical medium +customarily used for software interchange. +- b) Convey the object code in, or embodied in, a physical product +(including a physical distribution medium), accompanied by a +written offer, valid for at least three years and valid for as +long as you offer spare parts or customer support for that product +model, to give anyone who possesses the object code either (1) a +copy of the Corresponding Source for all the software in the +product that is covered by this License, on a durable physical +medium customarily used for software interchange, for a price no +more than your reasonable cost of physically performing this +conveying of source, or (2) access to copy the Corresponding +Source from a network server at no charge. +- c) Convey individual copies of the object code with a copy of the +written offer to provide the Corresponding Source. This +alternative is allowed only occasionally and noncommercially, and +only if you received the object code with such an offer, in accord +with subsection 6b. +- d) Convey the object code by offering access from a designated +place (gratis or for a charge), and offer equivalent access to the +Corresponding Source in the same way through the same place at no +further charge. You need not require recipients to copy the +Corresponding Source along with the object code. If the place to +copy the object code is a network server, the Corresponding Source +may be on a different server (operated by you or a third party) +that supports equivalent copying facilities, provided you maintain +clear directions next to the object code saying where to find the +Corresponding Source. Regardless of what server hosts the +Corresponding Source, you remain obligated to ensure that it is +available for as long as needed to satisfy these requirements. +- e) Convey the object code using peer-to-peer transmission, +provided you inform other peers where the object code and +Corresponding Source of the work are being offered to the general +public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + +A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, +family, or household purposes, or (2) anything designed or sold for +incorporation into a dwelling. In determining whether a product is a +consumer product, doubtful cases shall be resolved in favor of +coverage. For a particular product received by a particular user, +"normally used" refers to a typical or common use of that class of +product, regardless of the status of the particular user or of the way +in which the particular user actually uses, or expects or is expected +to use, the product. A product is a consumer product regardless of +whether the product has substantial commercial, industrial or +non-consumer uses, unless such uses represent the only significant +mode of use of the product. + +"Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to +install and execute modified versions of a covered work in that User +Product from a modified version of its Corresponding Source. The +information must suffice to ensure that the continued functioning of +the modified object code is in no case prevented or interfered with +solely because modification has been made. + +If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + +The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or +updates for a work that has been modified or installed by the +recipient, or for the User Product in which it has been modified or +installed. Access to a network may be denied when the modification +itself materially and adversely affects the operation of the network +or violates the rules and protocols for communication across the +network. + +Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + +#### 7. Additional Terms. + +"Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders +of that material) supplement the terms of this License with terms: + +- a) Disclaiming warranty or limiting liability differently from the +terms of sections 15 and 16 of this License; or +- b) Requiring preservation of specified reasonable legal notices or +author attributions in that material or in the Appropriate Legal +Notices displayed by works containing it; or +- c) Prohibiting misrepresentation of the origin of that material, +or requiring that modified versions of such material be marked in +reasonable ways as different from the original version; or +- d) Limiting the use for publicity purposes of names of licensors +or authors of the material; or +- e) Declining to grant rights under trademark law for use of some +trade names, trademarks, or service marks; or +- f) Requiring indemnification of licensors and authors of that +material by anyone who conveys the material (or modified versions +of it) with contractual assumptions of liability to the recipient, +for any liability that these contractual assumptions directly +impose on those licensors and authors. + +All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; the +above requirements apply either way. + +#### 8. Termination. + +You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + +However, if you cease all violation of this License, then your license +from a particular copyright holder is reinstated (a) provisionally, +unless and until the copyright holder explicitly and finally +terminates your license, and (b) permanently, if the copyright holder +fails to notify you of the violation by some reasonable means prior to +60 days after the cessation. + +Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + +#### 9. Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or run +a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + +#### 10. Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + +An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + +#### 11. Patents. + +A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + +A contributor's "essential patent claims" are all patent claims owned +or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + +In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + +If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + +A patent license is "discriminatory" if it does not include within the +scope of its coverage, prohibits the exercise of, or is conditioned on +the non-exercise of one or more of the rights that are specifically +granted under this License. You may not convey a covered work if you +are a party to an arrangement with a third party that is in the +business of distributing software, under which you make payment to the +third party based on the extent of your activity of conveying the +work, and under which the third party grants, to any of the parties +who would receive the covered work from you, a discriminatory patent +license (a) in connection with copies of the covered work conveyed by +you (or copies made from those copies), or (b) primarily for and in +connection with specific products or compilations that contain the +covered work, unless you entered into that arrangement, or that patent +license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + +#### 12. No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under +this License and any other pertinent obligations, then as a +consequence you may not convey it at all. For example, if you agree to +terms that obligate you to collect a royalty for further conveying +from those to whom you convey the Program, the only way you could +satisfy both those terms and this License would be to refrain entirely +from conveying the Program. + +#### 13. Use with the GNU Affero General Public License. + +Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + +#### 14. Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions +of the GNU General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in +detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies that a certain numbered version of the GNU General Public +License "or any later version" applies to it, you have the option of +following the terms and conditions either of that numbered version or +of any later version published by the Free Software Foundation. If the +Program does not specify a version number of the GNU General Public +License, you may choose any version ever published by the Free +Software Foundation. + +If the Program specifies that a proxy can decide which future versions +of the GNU General Public License can be used, that proxy's public +statement of acceptance of a version permanently authorizes you to +choose that version for the Program. + +Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + +#### 15. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT +WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND +PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE +DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR +CORRECTION. + +#### 16. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR +CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES +ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT +NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR +LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM +TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER +PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +#### 17. Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + +END OF TERMS AND CONDITIONS + +### How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these +terms. + +To do so, attach the following notices to the program. It is safest to +attach them to the start of each source file to most effectively state +the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + +Copyright (C) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Also add information on how to contact you by electronic and paper +mail. + +If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) +This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands \`show w' and \`show c' should show the +appropriate parts of the General Public License. Of course, your +program's commands might be different; for a GUI interface, you would +use an "about box". + +You should also get your employer (if you work as a programmer) or +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. For more information on this, and how to apply and follow +the GNU GPL, see . + +The GNU General Public License does not permit incorporating your +program into proprietary programs. If your program is a subroutine +library, you may consider it more useful to permit linking proprietary +applications with the library. If this is what you want to do, use the +GNU Lesser General Public License instead of this License. But first, +please read . diff --git a/README.md b/README.md new file mode 100644 index 0000000..8535b84 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +# NeoFS HTTP Gateway + +NeoFS HTTP Gateway is example of tool that provides basic interactions with NeoFS. +You can download files from NeoFS Network using NeoFS Gateway. + +## Install + +```go get -u github.com/nspcc-dev/neofs-gw``` + +## Configuration + +``` +# Environments: + +GW_REQUEST_TIMEOUT=Duration - timeout for request +GW_CONNECT_TIMEOUT=Duration - timeout for connection +GW_LISTEN_ADDRESS=host:port - address to listen connections +GW_NEOFS_NODE_ADDR=host:port - address of NeoFS Node +GW_KEEPALIVE_TIME=Duration - аfter a duration of this time if the client doesn't see any activity +it pings the server to see if the transport is still alive. +GW_KEEPALIVE_TIMEOUT=Duration - after having pinged for keepalive check, the client waits for a duration +of Timeout and if no activity is seen even after that the connection is closed +GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=Bool - if true, client sends keepalive pings even with no active RPCs. +If false, when there are no active RPCs, Time and Timeout will be ignored and no keepalive pings will be sent. +``` diff --git a/misc.go b/misc.go index 48700df..2977019 100644 --- a/misc.go +++ b/misc.go @@ -2,9 +2,4 @@ package main const DropInFilenameHeader = "drop-in-filename" -var ( - Prefix = "neofs" - Build = "now" - Version = "dev" - Debug = "true" -) +var Version = "dev" From 3f8e696433142242f6c7a2dd4c88e94fb19bb6fe Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 18 Nov 2019 19:59:59 +0300 Subject: [PATCH 004/548] update proto / crypto / makefile --- Makefile | 10 +++++----- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 8729d78..3abbeca 100644 --- a/Makefile +++ b/Makefile @@ -12,15 +12,15 @@ version: @echo $(VERSION) # Make sure that all files added to commit -vendor: +deps: @printf "${B}${G}⇒ Ensure vendor${R}: " - @GOPRIVATE=bitbucket.org/nspcc-dev go mod tidy -v && echo OK || (echo fail && exit 2) + @go mod tidy -v && echo OK || (echo fail && exit 2) @printf "${B}${G}⇒ Download requirements${R}: " - @GOPRIVATE=bitbucket.org/nspcc-dev go mod download && echo OK || (echo fail && exit 2) + @go mod download && echo OK || (echo fail && exit 2) @printf "${B}${G}⇒ Store vendor localy${R}: " - @GOPRIVATE=bitbucket.org/nspcc-dev go mod vendor && echo OK || (echo fail && exit 2) + @go mod vendor && echo OK || (echo fail && exit 2) -image: vendor +image: deps @echo "${B}${G}⇒ Build GW docker-image ${R}" @docker build \ --build-arg REPO=$(REPO) \ diff --git a/go.mod b/go.mod index 12b0157..52b7f84 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.13 require ( github.com/labstack/echo/v4 v4.1.11 github.com/nspcc-dev/hrw v1.0.9 // indirect - github.com/nspcc-dev/neofs-proto v0.0.0-20191101093315-0d5a92f60568 + github.com/nspcc-dev/neofs-proto v0.1.0 github.com/pkg/errors v0.8.1 github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.5.0 diff --git a/go.sum b/go.sum index 7d2e17d..d4e760c 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,8 @@ github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neofs-crypto v0.1.0 h1:G5Nz5AO2ALwRCFgQkgtifZZPfPqZtGd9/qu6OoEF9EA= github.com/nspcc-dev/neofs-crypto v0.1.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= +github.com/nspcc-dev/neofs-crypto v0.2.1/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= +github.com/nspcc-dev/neofs-proto v0.1.0/go.mod h1:dUbMyURyTeGx4tBz/Cz/ccsvQA6TRcNR1+EDB93Y4jA= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= github.com/nspcc-dev/netmap v1.6.1/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= github.com/nspcc-dev/rfc6979 v0.1.0 h1:Lwg7esRRoyK1Up/IN1vAef1EmvrBeMHeeEkek2fAJ6c= From 2bebf38a182dfe50b9bbfe31ccc8473603641b4b Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 19 Nov 2019 16:23:59 +0300 Subject: [PATCH 005/548] update go.sum --- go.sum | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/go.sum b/go.sum index d4e760c..9b652db 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -bitbucket.org/nspcc-dev/neofs-proto v0.0.0-20191101093315-0d5a92f60568 h1:Uy8dzzGOqK0QPJauQI2HseW2XwUIDwUq52mJjyrQqb8= -bitbucket.org/nspcc-dev/neofs-proto v0.0.0-20191101093315-0d5a92f60568/go.mod h1:SfhT/MhBkpJVCVS+hK/APVwHKtiEAnN9bNxEFJ2kx+E= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c h1:2RuXx1+tSNWRjxhY0Bx52kjV2odJQ0a6MTbfTPhGAkg= code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= @@ -110,9 +108,9 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neofs-crypto v0.1.0 h1:G5Nz5AO2ALwRCFgQkgtifZZPfPqZtGd9/qu6OoEF9EA= -github.com/nspcc-dev/neofs-crypto v0.1.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= +github.com/nspcc-dev/neofs-crypto v0.2.1 h1:NxKexcW88vlHO/u7EYjx5Q1UaOQ7XhYrCsLSVgOcCxw= github.com/nspcc-dev/neofs-crypto v0.2.1/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= +github.com/nspcc-dev/neofs-proto v0.1.0 h1:CloznEFNGhHrGR6MSsAREJOSpEGWBsJOmezLArJbfFo= github.com/nspcc-dev/neofs-proto v0.1.0/go.mod h1:dUbMyURyTeGx4tBz/Cz/ccsvQA6TRcNR1+EDB93Y4jA= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= github.com/nspcc-dev/netmap v1.6.1/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= From 97d1a99f58ce18960faad31063ef4dd756652bd6 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 13 Dec 2019 19:02:48 +0300 Subject: [PATCH 006/548] NSPCC-493 Separate repository for NeoFS GW - Update dependencies - Update README (configuration: flags section) - Add gRPC logger and flag to enable gRPC connection debugging - Refactored settings (add flags, use defaults instead of yaml representation) --- README.md | 14 ++++- go.mod | 17 +++-- go.sum | 96 +++++++++++++++++----------- logger.go | 64 +++++++++++++++++++ main.go | 178 +++++----------------------------------------------- misc.go | 5 +- receive.go | 139 ++++++++++++++++++++++++++++++++++++++++ settings.go | 121 +++++++++++++++++++++++++++++++++++ 8 files changed, 425 insertions(+), 209 deletions(-) create mode 100644 receive.go create mode 100644 settings.go diff --git a/README.md b/README.md index 8535b84..cc94508 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,24 @@ You can download files from NeoFS Network using NeoFS Gateway. ## Configuration ``` +# Flags + + -h, --help show help + -v, --version show version + --key string "gen" to generate key, path to private key file, hex string or wif (default "gen") + --verbose debug gRPC connections + --request_timeout duration gRPC request timeout (default 5s) + --connect_timeout duration gRPC connect timeout (default 30s) + --listen_address string HTTP Gateway listen address (default "0.0.0.0:8082") + --neofs_address string NeoFS Node address for proxying requests (default "0.0.0.0:8080") + # Environments: +GW_KEY=stirng - "gen" to generate key, path to private key file, hex string or wif (default "gen") GW_REQUEST_TIMEOUT=Duration - timeout for request GW_CONNECT_TIMEOUT=Duration - timeout for connection GW_LISTEN_ADDRESS=host:port - address to listen connections -GW_NEOFS_NODE_ADDR=host:port - address of NeoFS Node +GW_NEOFS_ADDRESS=host:port - address of NeoFS Node GW_KEEPALIVE_TIME=Duration - аfter a duration of this time if the client doesn't see any activity it pings the server to see if the transport is still alive. GW_KEEPALIVE_TIMEOUT=Duration - after having pinged for keepalive check, the client waits for a duration diff --git a/go.mod b/go.mod index 52b7f84..83a0e81 100644 --- a/go.mod +++ b/go.mod @@ -4,17 +4,14 @@ go 1.13 require ( github.com/labstack/echo/v4 v4.1.11 - github.com/nspcc-dev/hrw v1.0.9 // indirect - github.com/nspcc-dev/neofs-proto v0.1.0 + github.com/nspcc-dev/neofs-crypto v0.2.2 + github.com/nspcc-dev/neofs-proto v0.2.5 github.com/pkg/errors v0.8.1 - github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.5.0 - go.uber.org/zap v1.11.0 - golang.org/x/text v0.3.2 // indirect - google.golang.org/grpc v1.24.0 - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.6.1 + go.uber.org/zap v1.13.0 + google.golang.org/grpc v1.25.1 ) // Temporary, before we move repo to github: -replace github.com/nspcc-dev/neofs-proto v0.0.0-20191101093315-0d5a92f60568 => bitbucket.org/nspcc-dev/neofs-proto v0.0.0-20191101093315-0d5a92f60568 +// replace github.com/nspcc-dev/neofs-proto => ../neofs-proto diff --git a/go.sum b/go.sum index 9b652db..e52bed7 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c h1:2RuXx1+tSNWRjxhY0Bx52kjV2odJQ0a6MTbfTPhGAkg= -code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -16,6 +14,7 @@ github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -31,6 +30,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -58,19 +59,22 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -102,16 +106,16 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= -github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neofs-crypto v0.2.1 h1:NxKexcW88vlHO/u7EYjx5Q1UaOQ7XhYrCsLSVgOcCxw= -github.com/nspcc-dev/neofs-crypto v0.2.1/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= -github.com/nspcc-dev/neofs-proto v0.1.0 h1:CloznEFNGhHrGR6MSsAREJOSpEGWBsJOmezLArJbfFo= -github.com/nspcc-dev/neofs-proto v0.1.0/go.mod h1:dUbMyURyTeGx4tBz/Cz/ccsvQA6TRcNR1+EDB93Y4jA= +github.com/nspcc-dev/neofs-crypto v0.2.2 h1:jLc5O+Wdpaq7L4lNYFX7li+OP4I1FsvvcPW1NXm3erY= +github.com/nspcc-dev/neofs-crypto v0.2.2/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= +github.com/nspcc-dev/neofs-proto v0.2.5 h1:/zPystudzakARWvk618th4+4bYuipkFGSVdGAvAwRVo= +github.com/nspcc-dev/neofs-proto v0.2.5/go.mod h1:UDr4USXDHqn97StDvL7U/DcWRmRloIefaFo8qk4GbOo= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= github.com/nspcc-dev/netmap v1.6.1/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= github.com/nspcc-dev/rfc6979 v0.1.0 h1:Lwg7esRRoyK1Up/IN1vAef1EmvrBeMHeeEkek2fAJ6c= @@ -119,11 +123,6 @@ github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452 github.com/nspcc-dev/tzhash v1.3.0 h1:n6FTHsfPYbMi5Jmo6SwGVVRQD8i2w1P2ScCaW6rz69Q= github.com/nspcc-dev/tzhash v1.3.0/go.mod h1:Lc4DersKS8MNIrunTmsAzANO56qnG+LZ4GOE/WYGVzU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= -github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -148,8 +147,13 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -163,8 +167,8 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.5.0 h1:GpsTwfsQ27oS/Aha/6d1oD7tpKIqWnOA6tgOX9HHkt4= -github.com/spf13/viper v1.5.0/go.mod h1:AkYRkVJF8TkSG/xet6PzXX+l39KhhXa2pdqVSxnTcn4= +github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= +github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -182,29 +186,41 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.11.0 h1:gSmpCfs+R47a4yQPAI4xJ0IPDLTRGXskm6UelqNXpqE= -go.uber.org/zap v1.11.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -212,7 +228,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -223,34 +238,41 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -258,3 +280,5 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/logger.go b/logger.go index 07fa9b1..8807ab7 100644 --- a/logger.go +++ b/logger.go @@ -3,11 +3,17 @@ package main import ( "strings" + "google.golang.org/grpc/grpclog" + "github.com/spf13/viper" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) +type zapLogger struct { + log *zap.Logger +} + const ( formatJSON = "json" formatConsole = "console" @@ -16,6 +22,14 @@ const ( defaultSamplingThereafter = 100 ) +func gRPCLogger(l *zap.Logger) grpclog.LoggerV2 { + return &zapLogger{ + log: l.WithOptions( + // skip gRPCLog + zapLogger in caller + zap.AddCallerSkip(2)), + } +} + func safeLevel(lvl string) zap.AtomicLevel { switch strings.ToLower(lvl) { case "debug": @@ -87,3 +101,53 @@ func newLogger(v *viper.Viper) (*zap.Logger, error) { zap.String("app_name", name), zap.String("app_version", version)), nil } + +func (z *zapLogger) Info(args ...interface{}) { + z.log.Sugar().Info(args...) +} + +func (z *zapLogger) Infoln(args ...interface{}) { + z.log.Sugar().Info(args...) +} + +func (z *zapLogger) Infof(format string, args ...interface{}) { + z.log.Sugar().Infof(format, args...) +} + +func (z *zapLogger) Warning(args ...interface{}) { + z.log.Sugar().Warn(args...) +} + +func (z *zapLogger) Warningln(args ...interface{}) { + z.log.Sugar().Warn(args...) +} + +func (z *zapLogger) Warningf(format string, args ...interface{}) { + z.log.Sugar().Warnf(format, args...) +} + +func (z *zapLogger) Error(args ...interface{}) { + z.log.Sugar().Error(args...) +} + +func (z *zapLogger) Errorln(args ...interface{}) { + z.log.Sugar().Error(args...) +} + +func (z *zapLogger) Errorf(format string, args ...interface{}) { + z.log.Sugar().Errorf(format, args...) +} + +func (z *zapLogger) Fatal(args ...interface{}) { + z.log.Sugar().Fatal(args...) +} + +func (z *zapLogger) Fatalln(args ...interface{}) { + z.log.Sugar().Fatal(args...) +} + +func (z *zapLogger) Fatalf(format string, args ...interface{}) { + z.log.Sugar().Fatalf(format, args...) +} + +func (z *zapLogger) V(int) bool { return z.log.Core().Enabled(zapcore.DebugLevel) } diff --git a/main.go b/main.go index 77c4dc3..7a54993 100644 --- a/main.go +++ b/main.go @@ -2,65 +2,27 @@ package main import ( "context" - "io" - "net/http" - "strconv" - "strings" + "crypto/ecdsa" "time" "github.com/labstack/echo/v4" "github.com/nspcc-dev/neofs-proto/object" - "github.com/nspcc-dev/neofs-proto/refs" - "github.com/nspcc-dev/neofs-proto/service" - "github.com/pkg/errors" - "github.com/spf13/viper" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" "google.golang.org/grpc/keepalive" ) type config struct { log *zap.Logger timeout time.Duration + key *ecdsa.PrivateKey cli object.ServiceClient } -var defaultConfig = strings.NewReader(` -request_timeout: 5s -connect_timeout: 30s -listen_address: :8082 -neofs_node_addr: :8080 - -logger: - level: debug - format: console - trace_level: fatal - no_disclaimer: true - sampling: - initial: 1000 - thereafter: 1000 - -keepalive: - time: 100ms - timeout: 10s - permit_without_stream: true -`) - func main() { - v := viper.New() - v.AutomaticEnv() - v.SetEnvPrefix("GW") - v.SetConfigType("yaml") - v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - - // set defaults: - v.Set("app.name", "neofs-gw") - v.Set("app.version", Version) - - if err := v.ReadConfig(defaultConfig); err != nil { - panic(err) - } + v := settings() log, err := newLogger(v) if err != nil { @@ -74,12 +36,20 @@ func main() { grace = newGracefulContext(log) ) + if v.GetBool("verbose") { + grpclog.SetLoggerV2( + gRPCLogger(log)) + } + cfg.log = log + cfg.key = fetchKey(log, v) cfg.timeout = v.GetDuration("request_timeout") + ctx, cancel := context.WithTimeout(grace, v.GetDuration("connect_timeout")) defer cancel() - conn, err := grpc.DialContext(ctx, v.GetString("neofs_node_addr"), + conn, err := grpc.DialContext(ctx, v.GetString("neofs_address"), + grpc.WithBlock(), grpc.WithInsecure(), grpc.WithKeepaliveParams(keepalive.ClientParameters{ Time: v.GetDuration("keepalive.time"), @@ -105,9 +75,11 @@ func main() { e.HideBanner = true e.GET("/:cid/:oid", cfg.receiveFile) + go func() { log.Info("run gateway server", zap.String("address", v.GetString("listen_address"))) + if err := e.Start(v.GetString("listen_address")); err != nil { log.Panic("could not start server", zap.Error(err)) } @@ -118,9 +90,7 @@ func main() { ctx, cancel = context.WithTimeout(context.TODO(), time.Second*30) defer cancel() - if err := e.Shutdown(ctx); err != nil { - log.Panic("could not stop server", zap.Error(err)) - } + log.Info("stopping server", zap.Error(e.Shutdown(ctx))) } func checkConnection(ctx context.Context, conn *grpc.ClientConn, log *zap.Logger) { @@ -135,7 +105,7 @@ loop: case connectivity.Idle, connectivity.Connecting, connectivity.Ready: // It's ok.. default: - log.Panic("could not establish connection", + log.Error("could not establish connection", zap.Stringer("state", state), zap.Any("connection", conn.Target())) } @@ -144,117 +114,3 @@ loop: tick.Stop() } - -func (cfg *config) receiveFile(c echo.Context) error { - var ( - cid refs.CID - oid refs.ObjectID - obj *object.Object - download = c.QueryParam("download") != "" - ) - - cfg.log.Debug("try to fetch object from network", - zap.String("cid", c.Param("cid")), - zap.String("oid", c.Param("oid"))) - - if err := cid.Parse(c.Param("cid")); err != nil { - cfg.log.Error("wrong container id", - zap.String("cid", c.Param("cid")), - zap.String("oid", c.Param("oid")), - zap.Error(err)) - - return echo.NewHTTPError( - http.StatusBadRequest, - errors.Wrap(err, "wrong container id").Error(), - ) - } else if err := oid.Parse(c.Param("oid")); err != nil { - cfg.log.Error("wrong object id", - zap.Stringer("cid", cid), - zap.String("oid", c.Param("oid")), - zap.Error(err)) - - return echo.NewHTTPError( - http.StatusBadRequest, - errors.Wrap(err, "wrong object id").Error(), - ) - } - - ctx, cancel := context.WithTimeout(context.Background(), cfg.timeout) - defer cancel() - - cli, err := cfg.cli.Get(ctx, &object.GetRequest{ - Address: refs.Address{ObjectID: oid, CID: cid}, - TTL: service.SingleForwardingTTL, - }) - - if err != nil { - cfg.log.Error("could not prepare connection", - zap.Stringer("cid", cid), - zap.Stringer("oid", oid), - zap.Error(err)) - - return echo.NewHTTPError( - // TODO: nginx doesn't return 500 errors from backend - // http.StatusInternalServerError, - http.StatusBadRequest, - errors.Wrap(err, "could not prepare connection").Error(), - ) - } else if obj, err = receiveObject(cli); err != nil { - cfg.log.Error("could not receive object", - zap.Stringer("cid", cid), - zap.Stringer("oid", oid), - zap.Error(err)) - - if strings.Contains(err.Error(), object.ErrNotFound.Error()) { - return echo.NewHTTPError(http.StatusNotFound, err.Error()) - } - - return echo.NewHTTPError( - // TODO: nginx doesn't return 500 errors from backend - // http.StatusInternalServerError, - http.StatusBadRequest, - errors.Wrap(err, "could not receive object").Error(), - ) - } - - cfg.log.Info("object fetched successfully", - zap.Stringer("cid", cid), - zap.Stringer("oid", oid)) - - c.Response().Header().Set("Content-Length", strconv.FormatUint(obj.SystemHeader.PayloadLength, 10)) - c.Response().Header().Set("x-object-id", obj.SystemHeader.ID.String()) - c.Response().Header().Set("x-owner-id", obj.SystemHeader.OwnerID.String()) - c.Response().Header().Set("x-container-id", obj.SystemHeader.CID.String()) - - for i := range obj.Headers { - if hdr := obj.Headers[i].GetUserHeader(); hdr != nil { - c.Response().Header().Set("x-"+hdr.Key, hdr.Value) - - if hdr.Key == DropInFilenameHeader && download { - c.Response().Header().Set("Content-Disposition", "attachment; filename="+hdr.Value) - } - } - } - - return c.Blob(http.StatusOK, - http.DetectContentType(obj.Payload), - obj.Payload) -} - -func receiveObject(cli object.Service_GetClient) (*object.Object, error) { - var obj *object.Object - for { - resp, err := cli.Recv() - if err != nil { - if err == io.EOF { - break - } - return nil, err - } else if obj == nil { - obj = resp.GetObject() - } - - obj.Payload = append(obj.Payload, resp.GetChunk()...) - } - return obj, nil -} diff --git a/misc.go b/misc.go index 2977019..45f704d 100644 --- a/misc.go +++ b/misc.go @@ -2,4 +2,7 @@ package main const DropInFilenameHeader = "drop-in-filename" -var Version = "dev" +var ( + Build = "now" + Version = "dev" +) diff --git a/receive.go b/receive.go new file mode 100644 index 0000000..57cb666 --- /dev/null +++ b/receive.go @@ -0,0 +1,139 @@ +package main + +import ( + "context" + "io" + "net/http" + "strconv" + "strings" + + "github.com/labstack/echo/v4" + "github.com/nspcc-dev/neofs-proto/container" + "github.com/nspcc-dev/neofs-proto/object" + "github.com/nspcc-dev/neofs-proto/refs" + "github.com/nspcc-dev/neofs-proto/service" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +func (cfg *config) receiveFile(c echo.Context) error { + var ( + cid refs.CID + oid refs.ObjectID + obj *object.Object + download = c.QueryParam("download") != "" + ) + + cfg.log.Debug("try to fetch object from network", + zap.String("cid", c.Param("cid")), + zap.String("oid", c.Param("oid"))) + + if err := cid.Parse(c.Param("cid")); err != nil { + cfg.log.Error("wrong container id", + zap.String("cid", c.Param("cid")), + zap.String("oid", c.Param("oid")), + zap.Error(err)) + + return echo.NewHTTPError( + http.StatusBadRequest, + errors.Wrap(err, "wrong container id").Error(), + ) + } else if err := oid.Parse(c.Param("oid")); err != nil { + cfg.log.Error("wrong object id", + zap.Stringer("cid", cid), + zap.String("oid", c.Param("oid")), + zap.Error(err)) + + return echo.NewHTTPError( + http.StatusBadRequest, + errors.Wrap(err, "wrong object id").Error(), + ) + } + + ctx, cancel := context.WithTimeout(context.Background(), cfg.timeout) + defer cancel() + + req := &object.GetRequest{Address: refs.Address{ObjectID: oid, CID: cid}} + req.SetTTL(service.SingleForwardingTTL) + + if err := service.SignRequestHeader(cfg.key, req); err != nil { + cfg.log.Error("could not sign request", + zap.Stringer("cid", cid), + zap.Stringer("oid", oid), + zap.Error(err)) + + return echo.NewHTTPError( + http.StatusBadRequest, + errors.Wrap(err, "could not sign request").Error()) + } + + cli, err := cfg.cli.Get(ctx, req) + if err != nil { + cfg.log.Error("could not prepare connection", + zap.Stringer("cid", cid), + zap.Stringer("oid", oid), + zap.Error(err)) + + return echo.NewHTTPError( + http.StatusBadRequest, + errors.Wrap(err, "could not prepare connection").Error(), + ) + } else if obj, err = receiveObject(cli); err != nil { + cfg.log.Error("could not receive object", + zap.Stringer("cid", cid), + zap.Stringer("oid", oid), + zap.Error(err)) + + switch { + case strings.Contains(err.Error(), object.ErrNotFound.Error()), + strings.Contains(err.Error(), container.ErrNotFound.Error()): + return echo.NewHTTPError(http.StatusNotFound, err.Error()) + default: + return echo.NewHTTPError( + http.StatusBadRequest, + errors.Wrap(err, "could not receive object").Error(), + ) + } + } + + cfg.log.Info("object fetched successfully", + zap.Stringer("cid", cid), + zap.Stringer("oid", oid)) + + c.Response().Header().Set("Content-Length", strconv.FormatUint(obj.SystemHeader.PayloadLength, 10)) + c.Response().Header().Set("x-object-id", obj.SystemHeader.ID.String()) + c.Response().Header().Set("x-owner-id", obj.SystemHeader.OwnerID.String()) + c.Response().Header().Set("x-container-id", obj.SystemHeader.CID.String()) + + for i := range obj.Headers { + if hdr := obj.Headers[i].GetUserHeader(); hdr != nil { + c.Response().Header().Set("x-"+hdr.Key, hdr.Value) + + if hdr.Key == DropInFilenameHeader && download { + c.Response().Header().Set("Content-Disposition", "attachment; filename="+hdr.Value) + } + } + } + + return c.Blob(http.StatusOK, + http.DetectContentType(obj.Payload), + obj.Payload) +} + +func receiveObject(cli object.Service_GetClient) (*object.Object, error) { + var obj *object.Object + for { + resp, err := cli.Recv() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } else if obj == nil { + obj = resp.GetObject() + } + + obj.Payload = append(obj.Payload, resp.GetChunk()...) + } + return obj, nil +} diff --git a/settings.go b/settings.go new file mode 100644 index 0000000..c1f20f9 --- /dev/null +++ b/settings.go @@ -0,0 +1,121 @@ +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "fmt" + "io" + "os" + "strings" + "time" + + crypto "github.com/nspcc-dev/neofs-crypto" + "github.com/nspcc-dev/neofs-proto/refs" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "go.uber.org/zap" +) + +type empty int + +const ( + devNull = empty(0) + generate = "gen" +) + +func (empty) Read([]byte) (int, error) { return 0, io.EOF } + +func fetchKey(l *zap.Logger, v *viper.Viper) *ecdsa.PrivateKey { + switch val := v.GetString("key"); val { + case generate: + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + l.Fatal("could not generate private key", zap.Error(err)) + } + + id, err := refs.NewOwnerID(&key.PublicKey) + l.Info("generate new key", + zap.Stringer("key", id), + zap.Error(err)) + + return key + + default: + key, err := crypto.LoadPrivateKey(val) + if err != nil { + l.Fatal("could not load private key", + zap.String("key", v.GetString("key")), + zap.Error(err)) + } + + return key + } +} + +func settings() *viper.Viper { + v := viper.New() + v.AutomaticEnv() + v.SetEnvPrefix("GW") + v.SetConfigType("yaml") + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + + // flags setup: + flags := pflag.NewFlagSet("comandline", pflag.ExitOnError) + flags.SortFlags = false + + help := flags.BoolP("help", "h", false, "show help") + version := flags.BoolP("version", "v", false, "show version") + + flags.String("key", "gen", `"gen" to generate key, path to private key file, hex string or wif`) + + flags.Bool("verbose", false, "debug gRPC connections") + flags.Duration("request_timeout", time.Second*5, "gRPC request timeout") + flags.Duration("connect_timeout", time.Second*30, "gRPC connect timeout") + + flags.String("listen_address", "0.0.0.0:8082", "HTTP Gateway listen address") + flags.String("neofs_address", "0.0.0.0:8080", "NeoFS Node address for proxying requests") + + // set prefers: + v.Set("app.name", "neofs-gw") + v.Set("app.version", Version) + + // set defaults: + + // logger: + v.SetDefault("logger.level", "debug") + v.SetDefault("logger.format", "console") + v.SetDefault("logger.trace_level", "fatal") + v.SetDefault("logger.no_disclaimer", true) + v.SetDefault("logger.sampling.initial", 1000) + v.SetDefault("logger.sampling.thereafter", 1000) + + // keepalive: + v.SetDefault("keepalive.timeout", time.Second*10) + v.SetDefault("keepalive.time", time.Millisecond*100) + v.SetDefault("keepalive.permit_without_stream", true) + + if err := v.BindPFlags(flags); err != nil { + panic(err) + } + + if err := v.ReadConfig(devNull); err != nil { + panic(err) + } + + if err := flags.Parse(os.Args); err != nil { + panic(err) + } + + switch { + case help != nil && *help: + fmt.Printf("NeoFS HTTP Gateway %s (%s)\n", Version, Build) + flags.PrintDefaults() + os.Exit(0) + case version != nil && *version: + fmt.Printf("NeoFS HTTP Gateway %s (%s)\n", Version, Build) + os.Exit(0) + } + + return v +} From 798b9c9646a57ad17e86f645cf98e66346f29591 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 13 Dec 2019 19:18:45 +0300 Subject: [PATCH 007/548] docker: update dockerfile and make command --- Dockerfile | 6 ++++-- Makefile | 2 -- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9e28b96..fe791e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,13 +6,13 @@ ARG REPO=github.com/nspcc-dev/neofs-gw ENV GOGC off ENV CGO_ENABLED 0 -ENV LDFLAGS "-w -s -X ${REPO}/Version=${VERSION} -X ${REPO}/Build=${BUILD}" +ENV LDFLAGS "-w -s -X main.Version=${VERSION}" WORKDIR /src COPY . /src -RUN go build -v -mod=vendor -ldflags "${LDFLAGS}" -o /go/bin/neofs-gw ./ +RUN go build -v -mod=vendor -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N)" -o /go/bin/neofs-gw ./ # Executable image FROM scratch @@ -21,3 +21,5 @@ WORKDIR / COPY --from=builder /go/bin/neofs-gw /bin/neofs-gw COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + +ENTRYPOINT ["/bin/neofs-gw"] diff --git a/Makefile b/Makefile index 3abbeca..5721fe1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,3 @@ -REPO ?= $(shell go list -m) VERSION ?= "$(shell git describe --tags 2>/dev/null | sed 's/^v//')" HUB_IMAGE=nspccdev/neofs @@ -23,7 +22,6 @@ deps: image: deps @echo "${B}${G}⇒ Build GW docker-image ${R}" @docker build \ - --build-arg REPO=$(REPO) \ --build-arg VERSION=$(VERSION) \ -f Dockerfile \ -t $(HUB_IMAGE)-http-gate:$(VERSION) . From 39279e4c5fd68af9643e9e1391c48aca5683c570 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 17 Dec 2019 18:34:04 +0300 Subject: [PATCH 008/548] NSPCC-786 Rename `gen` => `generated` --- README.md | 4 ++-- settings.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cc94508..f080fd0 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ You can download files from NeoFS Network using NeoFS Gateway. -h, --help show help -v, --version show version - --key string "gen" to generate key, path to private key file, hex string or wif (default "gen") + --key string "generated" to generate key, path to private key file, hex string or wif (default "generated") --verbose debug gRPC connections --request_timeout duration gRPC request timeout (default 5s) --connect_timeout duration gRPC connect timeout (default 30s) @@ -23,7 +23,7 @@ You can download files from NeoFS Network using NeoFS Gateway. # Environments: -GW_KEY=stirng - "gen" to generate key, path to private key file, hex string or wif (default "gen") +GW_KEY=stirng - "generated" to generate key, path to private key file, hex string or wif (default "generated") GW_REQUEST_TIMEOUT=Duration - timeout for request GW_CONNECT_TIMEOUT=Duration - timeout for connection GW_LISTEN_ADDRESS=host:port - address to listen connections diff --git a/settings.go b/settings.go index c1f20f9..064013b 100644 --- a/settings.go +++ b/settings.go @@ -20,15 +20,15 @@ import ( type empty int const ( - devNull = empty(0) - generate = "gen" + devNull = empty(0) + generated = "generated" ) func (empty) Read([]byte) (int, error) { return 0, io.EOF } func fetchKey(l *zap.Logger, v *viper.Viper) *ecdsa.PrivateKey { switch val := v.GetString("key"); val { - case generate: + case generated: key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { l.Fatal("could not generate private key", zap.Error(err)) @@ -67,7 +67,7 @@ func settings() *viper.Viper { help := flags.BoolP("help", "h", false, "show help") version := flags.BoolP("version", "v", false, "show version") - flags.String("key", "gen", `"gen" to generate key, path to private key file, hex string or wif`) + flags.String("key", generated, `"`+generated+`" to generate key, path to private key file, hex string or wif`) flags.Bool("verbose", false, "debug gRPC connections") flags.Duration("request_timeout", time.Second*5, "gRPC request timeout") From 746946290c428ba1159ca99c43c325ea36839102 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Sat, 21 Dec 2019 13:26:14 +0300 Subject: [PATCH 009/548] NSPCC-762 Connection pool - implement connection pool - wait until container creates - refactoring gw service - add config option to enable prometheus and pprof - update neofs-proto to v0.2.8 --- Dockerfile | 2 +- README.md | 10 ++- go.mod | 3 +- go.sum | 10 +++ logger.go | 8 +- main.go | 99 +++++++++------------ pool.go | 244 ++++++++++++++++++++++++++++++++++++++++++++++++++++ receive.go | 46 ++++------ settings.go | 15 +++- 9 files changed, 337 insertions(+), 100 deletions(-) create mode 100644 pool.go diff --git a/Dockerfile b/Dockerfile index fe791e2..99cad69 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ WORKDIR /src COPY . /src -RUN go build -v -mod=vendor -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N)" -o /go/bin/neofs-gw ./ +RUN go build -v -mod=vendor -trimpath -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N)" -o /go/bin/neofs-gw ./ # Executable image FROM scratch diff --git a/README.md b/README.md index f080fd0..c79c586 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ You can download files from NeoFS Network using NeoFS Gateway. ``` # Flags - + --pprof enable pprof + --metrics enable prometheus -h, --help show help -v, --version show version --key string "generated" to generate key, path to private key file, hex string or wif (default "generated") @@ -19,7 +20,7 @@ You can download files from NeoFS Network using NeoFS Gateway. --request_timeout duration gRPC request timeout (default 5s) --connect_timeout duration gRPC connect timeout (default 30s) --listen_address string HTTP Gateway listen address (default "0.0.0.0:8082") - --neofs_address string NeoFS Node address for proxying requests (default "0.0.0.0:8080") + -p, --peers stringArray NeoFS nodes # Environments: @@ -27,7 +28,10 @@ GW_KEY=stirng - "generated" to generate key, path to p GW_REQUEST_TIMEOUT=Duration - timeout for request GW_CONNECT_TIMEOUT=Duration - timeout for connection GW_LISTEN_ADDRESS=host:port - address to listen connections -GW_NEOFS_ADDRESS=host:port - address of NeoFS Node +GW_PEERS__ADDRESS=host:port - address of NeoFS Node +GW_PEERS__WEIGHT=float - weight of NeoFS Node +GW_PPROF=bool - enable/disable pprof (/debug/pprof) +GW_METRICS=bool - enable/disable prometheus metrics endpoint (/metrics) GW_KEEPALIVE_TIME=Duration - аfter a duration of this time if the client doesn't see any activity it pings the server to see if the transport is still alive. GW_KEEPALIVE_TIMEOUT=Duration - after having pinged for keepalive check, the client waits for a duration diff --git a/go.mod b/go.mod index 83a0e81..112f041 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,9 @@ go 1.13 require ( github.com/labstack/echo/v4 v4.1.11 github.com/nspcc-dev/neofs-crypto v0.2.2 - github.com/nspcc-dev/neofs-proto v0.2.5 + github.com/nspcc-dev/neofs-proto v0.2.8 github.com/pkg/errors v0.8.1 + github.com/prometheus/client_golang v1.2.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.6.1 go.uber.org/zap v1.13.0 diff --git a/go.sum b/go.sum index e52bed7..c38f10d 100644 --- a/go.sum +++ b/go.sum @@ -13,9 +13,12 @@ github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310 h1:t+qxR github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -99,6 +102,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -116,6 +120,8 @@ github.com/nspcc-dev/neofs-crypto v0.2.2 h1:jLc5O+Wdpaq7L4lNYFX7li+OP4I1FsvvcPW1 github.com/nspcc-dev/neofs-crypto v0.2.2/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-proto v0.2.5 h1:/zPystudzakARWvk618th4+4bYuipkFGSVdGAvAwRVo= github.com/nspcc-dev/neofs-proto v0.2.5/go.mod h1:UDr4USXDHqn97StDvL7U/DcWRmRloIefaFo8qk4GbOo= +github.com/nspcc-dev/neofs-proto v0.2.8 h1:VWEIuqR2lQEuarkhLXhK62HLebSDmE6e4JqC7l2hc60= +github.com/nspcc-dev/neofs-proto v0.2.8/go.mod h1:uNc8tEAIcY828/kH94DpwiiSODvPHiesIM20hHckjAM= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= github.com/nspcc-dev/netmap v1.6.1/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= github.com/nspcc-dev/rfc6979 v0.1.0 h1:Lwg7esRRoyK1Up/IN1vAef1EmvrBeMHeeEkek2fAJ6c= @@ -133,17 +139,21 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= diff --git a/logger.go b/logger.go index 8807ab7..13aefe0 100644 --- a/logger.go +++ b/logger.go @@ -47,7 +47,7 @@ func safeLevel(lvl string) zap.AtomicLevel { } } -func newLogger(v *viper.Viper) (*zap.Logger, error) { +func newLogger(v *viper.Viper) *zap.Logger { c := zap.NewProductionConfig() c.OutputPaths = []string{"stdout"} @@ -87,11 +87,11 @@ func newLogger(v *viper.Viper) (*zap.Logger, error) { // enable trace only for current log-level zap.AddStacktrace(traceLvl)) if err != nil { - return nil, err + panic(err) } if v.GetBool("logger.no_disclaimer") { - return l, nil + return l } name := v.GetString("app.name") @@ -99,7 +99,7 @@ func newLogger(v *viper.Viper) (*zap.Logger, error) { return l.With( zap.String("app_name", name), - zap.String("app_version", version)), nil + zap.String("app_version", version)) } func (z *zapLogger) Info(args ...interface{}) { diff --git a/main.go b/main.go index 7a54993..4f59a47 100644 --- a/main.go +++ b/main.go @@ -3,114 +3,93 @@ package main import ( "context" "crypto/ecdsa" + "net/http" + _ "net/http/pprof" "time" "github.com/labstack/echo/v4" - "github.com/nspcc-dev/neofs-proto/object" + "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/zap" - "google.golang.org/grpc" - "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/keepalive" ) -type config struct { +type router struct { + pool *Pool log *zap.Logger timeout time.Duration key *ecdsa.PrivateKey - cli object.ServiceClient } func main() { - v := settings() - - log, err := newLogger(v) - if err != nil { - panic(err) - } - - log.Info("running application", zap.String("version", v.GetString("app.version"))) - var ( - cfg = new(config) - grace = newGracefulContext(log) + v = settings() + l = newLogger(v) + g = newGracefulContext(l) ) if v.GetBool("verbose") { - grpclog.SetLoggerV2( - gRPCLogger(log)) + grpclog.SetLoggerV2(gRPCLogger(l)) } - cfg.log = log - cfg.key = fetchKey(log, v) - cfg.timeout = v.GetDuration("request_timeout") - - ctx, cancel := context.WithTimeout(grace, v.GetDuration("connect_timeout")) - defer cancel() - - conn, err := grpc.DialContext(ctx, v.GetString("neofs_address"), - grpc.WithBlock(), - grpc.WithInsecure(), - grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: v.GetDuration("keepalive.time"), - Timeout: v.GetDuration("keepalive.timeout"), - PermitWithoutStream: v.GetBool("keepalive.permit_without_stream"), - }), - ) - if err != nil { - log.Panic("could not connect to neofs-node", - zap.String("neofs-node", v.GetString("neofs_node_addr")), - zap.Error(err)) + r := &router{ + log: l, + key: fetchKey(l, v), + pool: newPool(g, l, v), + timeout: v.GetDuration("request_timeout"), } - ctx, cancel = context.WithCancel(grace) - defer cancel() - - go checkConnection(ctx, conn, log) - cfg.cli = object.NewServiceClient(conn) + go checkConnection(g, r.pool) e := echo.New() e.Debug = false e.HidePort = true e.HideBanner = true - e.GET("/:cid/:oid", cfg.receiveFile) + e.GET("/:cid/:oid", r.receiveFile) + + // enable metrics + if v.GetBool("metrics") { + l.Info("enabled /metrics") + e.GET("/metrics", echo.WrapHandler(promhttp.Handler())) + } + + // enable pprof + if v.GetBool("pprof") { + l.Info("enabled /debug/pprof") + e.Any("/debug/pprof*", echo.WrapHandler(http.DefaultServeMux)) + } go func() { - log.Info("run gateway server", + l.Info("run gateway server", zap.String("address", v.GetString("listen_address"))) if err := e.Start(v.GetString("listen_address")); err != nil { - log.Panic("could not start server", zap.Error(err)) + l.Panic("could not start server", zap.Error(err)) } }() - <-ctx.Done() + <-g.Done() - ctx, cancel = context.WithTimeout(context.TODO(), time.Second*30) + ctx, cancel := context.WithTimeout(context.TODO(), time.Second*30) defer cancel() - log.Info("stopping server", zap.Error(e.Shutdown(ctx))) + l.Info("stopping server", zap.Error(e.Shutdown(ctx))) } -func checkConnection(ctx context.Context, conn *grpc.ClientConn, log *zap.Logger) { - tick := time.NewTicker(time.Second) +func checkConnection(ctx context.Context, p *Pool) { + tick := time.NewTicker(time.Second * 15) + loop: for { select { case <-ctx.Done(): break loop case <-tick.C: - switch state := conn.GetState(); state { - case connectivity.Idle, connectivity.Connecting, connectivity.Ready: - // It's ok.. - default: - log.Error("could not establish connection", - zap.Stringer("state", state), - zap.Any("connection", conn.Target())) - } + p.reBalance(ctx) } } tick.Stop() + + p.log.Info("stop connection worker") } diff --git a/pool.go b/pool.go new file mode 100644 index 0000000..fcc57a7 --- /dev/null +++ b/pool.go @@ -0,0 +1,244 @@ +package main + +import ( + "context" + crand "crypto/rand" + "encoding/binary" + "errors" + "math/rand" + "sort" + "strconv" + "sync" + "time" + + "github.com/spf13/viper" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/keepalive" +) + +type ( + node struct { + address string + weight uint32 + conn *grpc.ClientConn + } + + Pool struct { + log *zap.Logger + + ctl time.Duration + opts keepalive.ClientParameters + + cur *grpc.ClientConn + + *sync.RWMutex + nodes []*node + keys []uint32 + conns map[uint32][]*grpc.ClientConn + } +) + +var errNoHealthyConnections = errors.New("no active connections") + +func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) *Pool { + p := &Pool{ + log: l, + RWMutex: new(sync.RWMutex), + keys: make([]uint32, 0), + nodes: make([]*node, 0), + conns: make(map[uint32][]*grpc.ClientConn), + + // fill with defaults: + ctl: time.Second * 15, + opts: keepalive.ClientParameters{ + Time: time.Second * 10, + Timeout: time.Minute * 5, + PermitWithoutStream: true, + }, + } + buf := make([]byte, 8) + if _, err := crand.Read(buf); err != nil { + l.Panic("could not read seed", zap.Error(err)) + } + + seed := binary.BigEndian.Uint64(buf) + rand.Seed(int64(seed)) + l.Info("used random seed", zap.Uint64("seed", seed)) + + if val := v.GetDuration("connect_timeout"); val > 0 { + p.ctl = val + } + + if val := v.GetDuration("keepalive.time"); val > 0 { + p.opts.Time = val + } + + if val := v.GetDuration("keepalive.timeout"); val > 0 { + p.opts.Timeout = val + } + + if v.IsSet("keepalive.permit_without_stream") { + p.opts.PermitWithoutStream = v.GetBool("keepalive.permit_without_stream") + } + + for i := 0; ; i++ { + key := "peers." + strconv.Itoa(i) + "." + address := v.GetString(key + "address") + weight := v.GetFloat64(key + "weight") + + if address == "" { + l.Warn("skip, empty address") + break + } + + p.nodes = append(p.nodes, &node{ + address: address, + weight: uint32(weight * 100), + }) + + l.Info("add new peer", + zap.String("address", p.nodes[i].address), + zap.Uint32("weight", p.nodes[i].weight)) + } + + p.reBalance(ctx) + + cur, err := p.getConnection() + if err != nil { + l.Panic("could get connection", zap.Error(err)) + } + + p.cur = cur + + return p +} + +func (p *Pool) close() { + p.Lock() + defer p.Unlock() + + for i := range p.nodes { + if p.nodes[i] == nil || p.nodes[i].conn == nil { + continue + } + + p.log.Warn("close connection", + zap.String("address", p.nodes[i].address), + zap.Error(p.nodes[i].conn.Close())) + } +} + +func (p *Pool) reBalance(ctx context.Context) { + p.Lock() + defer p.Unlock() + + keys := make(map[uint32]struct{}) + + for i := range p.nodes { + var ( + idx = -1 + exists bool + err error + conn = p.nodes[i].conn + weight = p.nodes[i].weight + ) + + if conn == nil { + p.log.Warn("empty connection, try to connect", + zap.String("address", p.nodes[i].address)) + + ctx, cancel := context.WithTimeout(ctx, p.ctl) + conn, err = grpc.DialContext(ctx, p.nodes[i].address, + grpc.WithBlock(), + grpc.WithInsecure(), + grpc.WithKeepaliveParams(p.opts)) + cancel() + + if err != nil || conn == nil { + p.log.Warn("skip, could not connect to node", + zap.String("address", p.nodes[i].address), + zap.Error(err)) + continue + } + + p.nodes[i].conn = conn + p.log.Info("connected to node", zap.String("address", p.nodes[i].address)) + } + + switch st := conn.GetState(); st { + case connectivity.Idle, connectivity.Ready, connectivity.Connecting: + keys[weight] = struct{}{} + + for j := range p.conns[weight] { + if p.conns[weight][j] == conn { + exists = true + break + } + } + + if !exists { + p.conns[weight] = append(p.conns[weight], conn) + } + + // Problems with connection, try to remove from : + default: + for j := range p.conns[weight] { + if p.conns[weight][j] == conn { + idx = j + exists = true + break + } + } + + if exists { + // remove from connections + p.conns[weight] = append(p.conns[weight][:idx], p.conns[weight][idx+1:]...) + } + } + } + + p.keys = p.keys[:0] + for w := range keys { + p.keys = append(p.keys, w) + } + + sort.Slice(p.keys, func(i, j int) bool { + return p.keys[i] > p.keys[j] + }) +} + +func (p *Pool) getConnection() (*grpc.ClientConn, error) { + p.RLock() + defer p.RUnlock() + + if p.cur != nil && isAlive(p.cur) { + return p.cur, nil + } + + for _, w := range p.keys { + switch ln := len(p.conns[w]); ln { + case 0: + continue + case 1: + p.cur = p.conns[w][0] + return p.cur, nil + default: // > 1 + i := rand.Intn(ln) + p.cur = p.conns[w][i] + return p.cur, nil + } + } + + return nil, errNoHealthyConnections +} + +func isAlive(cur *grpc.ClientConn) bool { + switch st := cur.GetState(); st { + case connectivity.Idle, connectivity.Ready, connectivity.Connecting: + return true + default: + return false + } +} diff --git a/receive.go b/receive.go index 57cb666..d59372a 100644 --- a/receive.go +++ b/receive.go @@ -16,33 +16,33 @@ import ( "go.uber.org/zap" ) -func (cfg *config) receiveFile(c echo.Context) error { +func (r *router) receiveFile(c echo.Context) error { var ( cid refs.CID oid refs.ObjectID obj *object.Object download = c.QueryParam("download") != "" + con, err = r.pool.getConnection() ) - cfg.log.Debug("try to fetch object from network", + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + + log := r.log.With( + zap.String("node", con.Target()), zap.String("cid", c.Param("cid")), zap.String("oid", c.Param("oid"))) if err := cid.Parse(c.Param("cid")); err != nil { - cfg.log.Error("wrong container id", - zap.String("cid", c.Param("cid")), - zap.String("oid", c.Param("oid")), - zap.Error(err)) + log.Error("wrong container id", zap.Error(err)) return echo.NewHTTPError( http.StatusBadRequest, errors.Wrap(err, "wrong container id").Error(), ) } else if err := oid.Parse(c.Param("oid")); err != nil { - cfg.log.Error("wrong object id", - zap.Stringer("cid", cid), - zap.String("oid", c.Param("oid")), - zap.Error(err)) + log.Error("wrong object id", zap.Error(err)) return echo.NewHTTPError( http.StatusBadRequest, @@ -50,39 +50,29 @@ func (cfg *config) receiveFile(c echo.Context) error { ) } - ctx, cancel := context.WithTimeout(context.Background(), cfg.timeout) + ctx, cancel := context.WithTimeout(context.Background(), r.timeout) defer cancel() req := &object.GetRequest{Address: refs.Address{ObjectID: oid, CID: cid}} req.SetTTL(service.SingleForwardingTTL) - if err := service.SignRequestHeader(cfg.key, req); err != nil { - cfg.log.Error("could not sign request", - zap.Stringer("cid", cid), - zap.Stringer("oid", oid), - zap.Error(err)) - + if err := service.SignRequestHeader(r.key, req); err != nil { + log.Error("could not sign request", zap.Error(err)) return echo.NewHTTPError( http.StatusBadRequest, errors.Wrap(err, "could not sign request").Error()) } - cli, err := cfg.cli.Get(ctx, req) + cli, err := object.NewServiceClient(con).Get(ctx, req) if err != nil { - cfg.log.Error("could not prepare connection", - zap.Stringer("cid", cid), - zap.Stringer("oid", oid), - zap.Error(err)) + log.Error("could not prepare connection", zap.Error(err)) return echo.NewHTTPError( http.StatusBadRequest, errors.Wrap(err, "could not prepare connection").Error(), ) } else if obj, err = receiveObject(cli); err != nil { - cfg.log.Error("could not receive object", - zap.Stringer("cid", cid), - zap.Stringer("oid", oid), - zap.Error(err)) + log.Error("could not receive object", zap.Error(err)) switch { case strings.Contains(err.Error(), object.ErrNotFound.Error()), @@ -96,9 +86,7 @@ func (cfg *config) receiveFile(c echo.Context) error { } } - cfg.log.Info("object fetched successfully", - zap.Stringer("cid", cid), - zap.Stringer("oid", oid)) + log.Info("object fetched successfully") c.Response().Header().Set("Content-Length", strconv.FormatUint(obj.SystemHeader.PayloadLength, 10)) c.Response().Header().Set("x-object-id", obj.SystemHeader.ID.String()) diff --git a/settings.go b/settings.go index 064013b..472850e 100644 --- a/settings.go +++ b/settings.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" "time" @@ -61,9 +62,12 @@ func settings() *viper.Viper { v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) // flags setup: - flags := pflag.NewFlagSet("comandline", pflag.ExitOnError) + flags := pflag.NewFlagSet("commandline", pflag.ExitOnError) flags.SortFlags = false + flags.Bool("pprof", false, "enable pprof") + flags.Bool("metrics", false, "enable prometheus") + help := flags.BoolP("help", "h", false, "show help") version := flags.BoolP("version", "v", false, "show version") @@ -74,7 +78,7 @@ func settings() *viper.Viper { flags.Duration("connect_timeout", time.Second*30, "gRPC connect timeout") flags.String("listen_address", "0.0.0.0:8082", "HTTP Gateway listen address") - flags.String("neofs_address", "0.0.0.0:8080", "NeoFS Node address for proxying requests") + peers := flags.StringArrayP("peers", "p", nil, "NeoFS nodes") // set prefers: v.Set("app.name", "neofs-gw") @@ -117,5 +121,12 @@ func settings() *viper.Viper { os.Exit(0) } + if peers != nil && len(*peers) > 0 { + for i := range *peers { + v.SetDefault("peers."+strconv.Itoa(i)+".address", (*peers)[i]) + v.SetDefault("peers."+strconv.Itoa(i)+".weight", 1) + } + } + return v } From d807ede7c721f69682b9616453607adc92d00772 Mon Sep 17 00:00:00 2001 From: alexvanin Date: Wed, 22 Jan 2020 17:24:15 +0300 Subject: [PATCH 010/548] dep: Update neofs-proto to v0.2.11 Object get and head requests have new `raw` field now. --- go.mod | 4 ++-- go.sum | 15 +++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 112f041..856bdd0 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.13 require ( github.com/labstack/echo/v4 v4.1.11 - github.com/nspcc-dev/neofs-crypto v0.2.2 - github.com/nspcc-dev/neofs-proto v0.2.8 + github.com/nspcc-dev/neofs-crypto v0.2.3 + github.com/nspcc-dev/neofs-proto v0.2.11 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.2.1 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index c38f10d..79d6f5c 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,7 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -116,16 +117,14 @@ github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neofs-crypto v0.2.2 h1:jLc5O+Wdpaq7L4lNYFX7li+OP4I1FsvvcPW1NXm3erY= -github.com/nspcc-dev/neofs-crypto v0.2.2/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= -github.com/nspcc-dev/neofs-proto v0.2.5 h1:/zPystudzakARWvk618th4+4bYuipkFGSVdGAvAwRVo= -github.com/nspcc-dev/neofs-proto v0.2.5/go.mod h1:UDr4USXDHqn97StDvL7U/DcWRmRloIefaFo8qk4GbOo= -github.com/nspcc-dev/neofs-proto v0.2.8 h1:VWEIuqR2lQEuarkhLXhK62HLebSDmE6e4JqC7l2hc60= -github.com/nspcc-dev/neofs-proto v0.2.8/go.mod h1:uNc8tEAIcY828/kH94DpwiiSODvPHiesIM20hHckjAM= +github.com/nspcc-dev/neofs-crypto v0.2.3 h1:aca3X2aly92ENRbFK+kH6Hd+J9EQ4Eu6XMVoITSIKtc= +github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= +github.com/nspcc-dev/neofs-proto v0.2.11 h1:+ahFZZE5E8V1L/Hhph7KC2YslXApy3fEjH88W3xKNJ0= +github.com/nspcc-dev/neofs-proto v0.2.11/go.mod h1:1bZzjSyISOzN0khssNO2lqVAGa9YHkN2jUvu8OHaF8s= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= github.com/nspcc-dev/netmap v1.6.1/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= -github.com/nspcc-dev/rfc6979 v0.1.0 h1:Lwg7esRRoyK1Up/IN1vAef1EmvrBeMHeeEkek2fAJ6c= -github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= +github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= +github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/tzhash v1.3.0 h1:n6FTHsfPYbMi5Jmo6SwGVVRQD8i2w1P2ScCaW6rz69Q= github.com/nspcc-dev/tzhash v1.3.0/go.mod h1:Lc4DersKS8MNIrunTmsAzANO56qnG+LZ4GOE/WYGVzU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= From 590444459d55ca4acc2e8537e01dbf3b1f078692 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 4 Feb 2020 13:32:38 +0300 Subject: [PATCH 011/548] Migrate to NeoFS API --- go.mod | 2 +- go.sum | 6 ++++-- receive.go | 8 ++++---- settings.go | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 856bdd0..93f9a1b 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.13 require ( github.com/labstack/echo/v4 v4.1.11 + github.com/nspcc-dev/neofs-api v0.2.13 github.com/nspcc-dev/neofs-crypto v0.2.3 - github.com/nspcc-dev/neofs-proto v0.2.11 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.2.1 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 79d6f5c..1ca75a0 100644 --- a/go.sum +++ b/go.sum @@ -117,13 +117,15 @@ github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= +github.com/nspcc-dev/neofs-api v0.2.13 h1:bcHFW7uSFHoQ8jd/+ji7zVBibi4KsvXCWlNHrWLuduM= +github.com/nspcc-dev/neofs-api v0.2.13/go.mod h1:fbObruAfxjtWt9BKQEFnKUauXDXEdeEWFLex+5UQyyc= github.com/nspcc-dev/neofs-crypto v0.2.3 h1:aca3X2aly92ENRbFK+kH6Hd+J9EQ4Eu6XMVoITSIKtc= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-proto v0.2.11 h1:+ahFZZE5E8V1L/Hhph7KC2YslXApy3fEjH88W3xKNJ0= -github.com/nspcc-dev/neofs-proto v0.2.11/go.mod h1:1bZzjSyISOzN0khssNO2lqVAGa9YHkN2jUvu8OHaF8s= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= github.com/nspcc-dev/netmap v1.6.1/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= +github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= +github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/tzhash v1.3.0 h1:n6FTHsfPYbMi5Jmo6SwGVVRQD8i2w1P2ScCaW6rz69Q= github.com/nspcc-dev/tzhash v1.3.0/go.mod h1:Lc4DersKS8MNIrunTmsAzANO56qnG+LZ4GOE/WYGVzU= diff --git a/receive.go b/receive.go index d59372a..1c50a7e 100644 --- a/receive.go +++ b/receive.go @@ -8,10 +8,10 @@ import ( "strings" "github.com/labstack/echo/v4" - "github.com/nspcc-dev/neofs-proto/container" - "github.com/nspcc-dev/neofs-proto/object" - "github.com/nspcc-dev/neofs-proto/refs" - "github.com/nspcc-dev/neofs-proto/service" + "github.com/nspcc-dev/neofs-api/container" + "github.com/nspcc-dev/neofs-api/object" + "github.com/nspcc-dev/neofs-api/refs" + "github.com/nspcc-dev/neofs-api/service" "github.com/pkg/errors" "go.uber.org/zap" ) diff --git a/settings.go b/settings.go index 472850e..50f8774 100644 --- a/settings.go +++ b/settings.go @@ -11,8 +11,8 @@ import ( "strings" "time" + "github.com/nspcc-dev/neofs-api/refs" crypto "github.com/nspcc-dev/neofs-crypto" - "github.com/nspcc-dev/neofs-proto/refs" "github.com/spf13/pflag" "github.com/spf13/viper" "go.uber.org/zap" From 461a4acea05a288767a3801b1f4332fe0bd34f2e Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 4 Feb 2020 14:00:06 +0300 Subject: [PATCH 012/548] Update to NeoFS API v0.2.14 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 93f9a1b..8ab2f0d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/labstack/echo/v4 v4.1.11 - github.com/nspcc-dev/neofs-api v0.2.13 + github.com/nspcc-dev/neofs-api v0.2.14 github.com/nspcc-dev/neofs-crypto v0.2.3 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.2.1 diff --git a/go.sum b/go.sum index 1ca75a0..7878191 100644 --- a/go.sum +++ b/go.sum @@ -117,8 +117,8 @@ github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neofs-api v0.2.13 h1:bcHFW7uSFHoQ8jd/+ji7zVBibi4KsvXCWlNHrWLuduM= -github.com/nspcc-dev/neofs-api v0.2.13/go.mod h1:fbObruAfxjtWt9BKQEFnKUauXDXEdeEWFLex+5UQyyc= +github.com/nspcc-dev/neofs-api v0.2.14 h1:vshzmwgkTHLzYCY+xoa43V4WY6TkKcWfxM0Qb6Hwgs0= +github.com/nspcc-dev/neofs-api v0.2.14/go.mod h1:fbObruAfxjtWt9BKQEFnKUauXDXEdeEWFLex+5UQyyc= github.com/nspcc-dev/neofs-crypto v0.2.3 h1:aca3X2aly92ENRbFK+kH6Hd+J9EQ4Eu6XMVoITSIKtc= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= From 53244fcaf30d5f922d9a81c70050a8b3b1cba66a Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 4 Feb 2020 14:00:45 +0300 Subject: [PATCH 013/548] Use object.FilenameHeader instead custom constant --- misc.go | 2 -- receive.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/misc.go b/misc.go index 45f704d..a2bc3df 100644 --- a/misc.go +++ b/misc.go @@ -1,7 +1,5 @@ package main -const DropInFilenameHeader = "drop-in-filename" - var ( Build = "now" Version = "dev" diff --git a/receive.go b/receive.go index 1c50a7e..319c7ea 100644 --- a/receive.go +++ b/receive.go @@ -97,7 +97,7 @@ func (r *router) receiveFile(c echo.Context) error { if hdr := obj.Headers[i].GetUserHeader(); hdr != nil { c.Response().Header().Set("x-"+hdr.Key, hdr.Value) - if hdr.Key == DropInFilenameHeader && download { + if hdr.Key == object.FilenameHeader && download { c.Response().Header().Set("Content-Disposition", "attachment; filename="+hdr.Value) } } From 1542a01924ca9e8d3752b407775fed2fb0bd1500 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 4 Feb 2020 14:02:29 +0300 Subject: [PATCH 014/548] Use path.Base because hdr.Value can be something like `/path/to/filename.ext` --- receive.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/receive.go b/receive.go index 319c7ea..78c79c1 100644 --- a/receive.go +++ b/receive.go @@ -4,6 +4,7 @@ import ( "context" "io" "net/http" + "path" "strconv" "strings" @@ -98,7 +99,8 @@ func (r *router) receiveFile(c echo.Context) error { c.Response().Header().Set("x-"+hdr.Key, hdr.Value) if hdr.Key == object.FilenameHeader && download { - c.Response().Header().Set("Content-Disposition", "attachment; filename="+hdr.Value) + // NOTE: we use path.Base because hdr.Value can be something like `/path/to/filename.ext` + c.Response().Header().Set("Content-Disposition", "attachment; filename="+path.Base(hdr.Value)) } } } From f14401c0c0c5b8e0161e3e1285e23d6acddd8d3c Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 7 Feb 2020 18:04:02 +0300 Subject: [PATCH 015/548] Update dependencies - github.com/labstack/echo/v4 `v4.1.11 => v4.1.14` - github.com/nspcc-dev/neofs-api `v0.2.14 => v0.3.1` - github.com/pkg/errors `v0.8.1 => v0.9.1` - github.com/prometheus/client_golang `v1.2.1 => v1.4.1` - github.com/spf13/viper `v1.6.1 => v1.6.2` - google.golang.org/grpc `v1.25.1 => v1.27.1` --- go.mod | 12 ++++++------ go.sum | 31 +++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 8ab2f0d..528358f 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,15 @@ module github.com/nspcc-dev/neofs-gw go 1.13 require ( - github.com/labstack/echo/v4 v4.1.11 - github.com/nspcc-dev/neofs-api v0.2.14 + github.com/labstack/echo/v4 v4.1.14 // v4.1.11 => v4.1.14 + github.com/nspcc-dev/neofs-api v0.3.1 // v0.2.14 => v0.3.1 github.com/nspcc-dev/neofs-crypto v0.2.3 - github.com/pkg/errors v0.8.1 - github.com/prometheus/client_golang v1.2.1 + github.com/pkg/errors v0.9.1 // v0.8.1 => v0.9.1 + github.com/prometheus/client_golang v1.4.1 // v1.2.1 => v1.4.1 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.6.1 + github.com/spf13/viper v1.6.2 // v1.6.1 => v1.6.2 go.uber.org/zap v1.13.0 - google.golang.org/grpc v1.25.1 + google.golang.org/grpc v1.27.1 // v1.25.1 => v1.27.1 ) // Temporary, before we move repo to github: diff --git a/go.sum b/go.sum index 7878191..fc60c4a 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,7 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -34,6 +35,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= @@ -58,10 +60,13 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= @@ -77,6 +82,7 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -92,6 +98,7 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/labstack/echo/v4 v4.1.11 h1:z0BZoArY4FqdpUEl+wlHp4hnr/oSR6MTmQmv8OHSoww= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/echo/v4 v4.1.14/go.mod h1:Q5KZ1vD3V5FEzjM79hjwVrC3ABr7F5IdM23bXQMRDGg= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= @@ -99,10 +106,12 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= @@ -119,6 +128,7 @@ github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neofs-api v0.2.14 h1:vshzmwgkTHLzYCY+xoa43V4WY6TkKcWfxM0Qb6Hwgs0= github.com/nspcc-dev/neofs-api v0.2.14/go.mod h1:fbObruAfxjtWt9BKQEFnKUauXDXEdeEWFLex+5UQyyc= +github.com/nspcc-dev/neofs-api v0.3.1/go.mod h1:Y1PQ6nEZyA6sZPDHJxgPWWeT1oU2bKL1N0rpFQ4B/fk= github.com/nspcc-dev/neofs-crypto v0.2.3 h1:aca3X2aly92ENRbFK+kH6Hd+J9EQ4Eu6XMVoITSIKtc= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= @@ -129,12 +139,14 @@ github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452 github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/tzhash v1.3.0 h1:n6FTHsfPYbMi5Jmo6SwGVVRQD8i2w1P2ScCaW6rz69Q= github.com/nspcc-dev/tzhash v1.3.0/go.mod h1:Lc4DersKS8MNIrunTmsAzANO56qnG+LZ4GOE/WYGVzU= +github.com/nspcc-dev/tzhash v1.4.0/go.mod h1:Z8gp/VZbyWgPhaMp/KTmeoW5UTynp/N60g0jTtSzBws= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -142,20 +154,24 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -180,6 +196,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -194,6 +211,7 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -214,6 +232,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -232,11 +251,13 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -249,10 +270,16 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -264,6 +291,7 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -275,11 +303,13 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -289,6 +319,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= From b8a6af43aa7f188dc5deb7b0fb6aca31ef4fe525 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 7 Feb 2020 18:06:19 +0300 Subject: [PATCH 016/548] update go.sum --- go.sum | 49 +++++++++++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/go.sum b/go.sum index fc60c4a..d256e10 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= -github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -34,7 +33,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -60,12 +58,12 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -81,7 +79,6 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -96,8 +93,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/labstack/echo/v4 v4.1.11 h1:z0BZoArY4FqdpUEl+wlHp4hnr/oSR6MTmQmv8OHSoww= -github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/echo/v4 v4.1.14 h1:h8XP66UfB3tUm+L3QPw7tmwAu3pJaA/nyfHPCcz46ic= github.com/labstack/echo/v4 v4.1.14/go.mod h1:Q5KZ1vD3V5FEzjM79hjwVrC3ABr7F5IdM23bXQMRDGg= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= @@ -106,11 +102,13 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -126,8 +124,7 @@ github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neofs-api v0.2.14 h1:vshzmwgkTHLzYCY+xoa43V4WY6TkKcWfxM0Qb6Hwgs0= -github.com/nspcc-dev/neofs-api v0.2.14/go.mod h1:fbObruAfxjtWt9BKQEFnKUauXDXEdeEWFLex+5UQyyc= +github.com/nspcc-dev/neofs-api v0.3.1 h1:QbaTcBTUXWjihMR5kw+UPvCjzmLx6g5m3qDvyhyBmzE= github.com/nspcc-dev/neofs-api v0.3.1/go.mod h1:Y1PQ6nEZyA6sZPDHJxgPWWeT1oU2bKL1N0rpFQ4B/fk= github.com/nspcc-dev/neofs-crypto v0.2.3 h1:aca3X2aly92ENRbFK+kH6Hd+J9EQ4Eu6XMVoITSIKtc= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= @@ -137,8 +134,7 @@ github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYv github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= -github.com/nspcc-dev/tzhash v1.3.0 h1:n6FTHsfPYbMi5Jmo6SwGVVRQD8i2w1P2ScCaW6rz69Q= -github.com/nspcc-dev/tzhash v1.3.0/go.mod h1:Lc4DersKS8MNIrunTmsAzANO56qnG+LZ4GOE/WYGVzU= +github.com/nspcc-dev/tzhash v1.4.0 h1:RVIR+mxOBHl58CE99+DXtE31ylD5PEkZSoWqoj4fVjg= github.com/nspcc-dev/tzhash v1.4.0/go.mod h1:Z8gp/VZbyWgPhaMp/KTmeoW5UTynp/N60g0jTtSzBws= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= @@ -146,31 +142,30 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= -github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= +github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -194,8 +189,7 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= -github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -211,6 +205,7 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4= github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -229,9 +224,9 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -251,6 +246,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -268,14 +264,14 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -291,6 +287,7 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -300,15 +297,14 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= -google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= @@ -319,6 +315,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From fbfab8cae76130aa74f7813862668c1ba6a275ea Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 13 Feb 2020 17:31:33 +0300 Subject: [PATCH 017/548] Add state.HealthCheck to isActive --- pool.go | 26 +++++++++++++++++++++----- receive.go | 5 +++-- settings.go | 2 +- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/pool.go b/pool.go index fcc57a7..1fdcc8d 100644 --- a/pool.go +++ b/pool.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/nspcc-dev/neofs-api/state" "github.com/spf13/viper" "go.uber.org/zap" "google.golang.org/grpc" @@ -105,7 +106,7 @@ func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) *Pool { p.reBalance(ctx) - cur, err := p.getConnection() + cur, err := p.getConnection(ctx) if err != nil { l.Panic("could get connection", zap.Error(err)) } @@ -196,6 +197,14 @@ func (p *Pool) reBalance(ctx context.Context) { // remove from connections p.conns[weight] = append(p.conns[weight][:idx], p.conns[weight][idx+1:]...) } + + if err := conn.Close(); err != nil { + p.log.Warn("could not close bad connection", zap.Error(err)) + } + + if p.nodes[i].conn != nil { + p.nodes[i].conn = nil + } } } @@ -209,11 +218,11 @@ func (p *Pool) reBalance(ctx context.Context) { }) } -func (p *Pool) getConnection() (*grpc.ClientConn, error) { +func (p *Pool) getConnection(ctx context.Context) (*grpc.ClientConn, error) { p.RLock() defer p.RUnlock() - if p.cur != nil && isAlive(p.cur) { + if p.cur != nil && isAlive(ctx, p.log, p.cur) { return p.cur, nil } @@ -234,10 +243,17 @@ func (p *Pool) getConnection() (*grpc.ClientConn, error) { return nil, errNoHealthyConnections } -func isAlive(cur *grpc.ClientConn) bool { +func isAlive(ctx context.Context, log *zap.Logger, cur *grpc.ClientConn) bool { switch st := cur.GetState(); st { case connectivity.Idle, connectivity.Ready, connectivity.Connecting: - return true + res, err := state.NewStatusClient(cur).HealthCheck(ctx, new(state.HealthRequest)) + if err != nil { + log.Warn("could not fetch health-check", zap.Error(err)) + + return false + } + + return res.Healthy default: return false } diff --git a/receive.go b/receive.go index 78c79c1..136156e 100644 --- a/receive.go +++ b/receive.go @@ -22,8 +22,9 @@ func (r *router) receiveFile(c echo.Context) error { cid refs.CID oid refs.ObjectID obj *object.Object + ctx = c.Request().Context() + con, err = r.pool.getConnection(ctx) download = c.QueryParam("download") != "" - con, err = r.pool.getConnection() ) if err != nil { @@ -51,7 +52,7 @@ func (r *router) receiveFile(c echo.Context) error { ) } - ctx, cancel := context.WithTimeout(context.Background(), r.timeout) + ctx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() req := &object.GetRequest{Address: refs.Address{ObjectID: oid, CID: cid}} diff --git a/settings.go b/settings.go index 50f8774..fa024fe 100644 --- a/settings.go +++ b/settings.go @@ -96,7 +96,7 @@ func settings() *viper.Viper { // keepalive: v.SetDefault("keepalive.timeout", time.Second*10) - v.SetDefault("keepalive.time", time.Millisecond*100) + v.SetDefault("keepalive.time", time.Second*10) // If set below 10s, a minimum value of 10s will be used instead. v.SetDefault("keepalive.permit_without_stream", true) if err := v.BindPFlags(flags); err != nil { From 17768b5bfeb76785e943af700a70e779552a7c0b Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 14 Feb 2020 12:36:31 +0300 Subject: [PATCH 018/548] set ttl to HealthRequest --- pool.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pool.go b/pool.go index 1fdcc8d..e0c3eaa 100644 --- a/pool.go +++ b/pool.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/nspcc-dev/neofs-api/service" "github.com/nspcc-dev/neofs-api/state" "github.com/spf13/viper" "go.uber.org/zap" @@ -246,7 +247,10 @@ func (p *Pool) getConnection(ctx context.Context) (*grpc.ClientConn, error) { func isAlive(ctx context.Context, log *zap.Logger, cur *grpc.ClientConn) bool { switch st := cur.GetState(); st { case connectivity.Idle, connectivity.Ready, connectivity.Connecting: - res, err := state.NewStatusClient(cur).HealthCheck(ctx, new(state.HealthRequest)) + req := new(state.HealthRequest) + req.SetTTL(service.NonForwardingTTL) + + res, err := state.NewStatusClient(cur).HealthCheck(ctx, req) if err != nil { log.Warn("could not fetch health-check", zap.Error(err)) From 01e152774f0d3189df8c28661a1960508a6edee9 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 14 Feb 2020 13:05:51 +0300 Subject: [PATCH 019/548] refactor reBalance and isAlive --- pool.go | 81 ++++++++++++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/pool.go b/pool.go index e0c3eaa..d361881 100644 --- a/pool.go +++ b/pool.go @@ -30,8 +30,8 @@ type ( Pool struct { log *zap.Logger - ctl time.Duration - opts keepalive.ClientParameters + connectTimeout time.Duration + opts keepalive.ClientParameters cur *grpc.ClientConn @@ -42,7 +42,10 @@ type ( } ) -var errNoHealthyConnections = errors.New("no active connections") +var ( + errEmptyConnection = errors.New("empty connection") + errNoHealthyConnections = errors.New("no active connections") +) func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) *Pool { p := &Pool{ @@ -53,7 +56,7 @@ func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) *Pool { conns: make(map[uint32][]*grpc.ClientConn), // fill with defaults: - ctl: time.Second * 15, + connectTimeout: time.Second * 15, opts: keepalive.ClientParameters{ Time: time.Second * 10, Timeout: time.Minute * 5, @@ -70,7 +73,7 @@ func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) *Pool { l.Info("used random seed", zap.Uint64("seed", seed)) if val := v.GetDuration("connect_timeout"); val > 0 { - p.ctl = val + p.connectTimeout = val } if val := v.GetDuration("keepalive.time"); val > 0 { @@ -138,11 +141,14 @@ func (p *Pool) reBalance(ctx context.Context) { keys := make(map[uint32]struct{}) + p.log.Info("re-balancing connections") + for i := range p.nodes { var ( idx = -1 exists bool err error + start = time.Now() conn = p.nodes[i].conn weight = p.nodes[i].weight ) @@ -151,7 +157,7 @@ func (p *Pool) reBalance(ctx context.Context) { p.log.Warn("empty connection, try to connect", zap.String("address", p.nodes[i].address)) - ctx, cancel := context.WithTimeout(ctx, p.ctl) + ctx, cancel := context.WithTimeout(ctx, p.connectTimeout) conn, err = grpc.DialContext(ctx, p.nodes[i].address, grpc.WithBlock(), grpc.WithInsecure(), @@ -161,6 +167,7 @@ func (p *Pool) reBalance(ctx context.Context) { if err != nil || conn == nil { p.log.Warn("skip, could not connect to node", zap.String("address", p.nodes[i].address), + zap.Duration("elapsed", time.Since(start)), zap.Error(err)) continue } @@ -169,43 +176,41 @@ func (p *Pool) reBalance(ctx context.Context) { p.log.Info("connected to node", zap.String("address", p.nodes[i].address)) } - switch st := conn.GetState(); st { - case connectivity.Idle, connectivity.Ready, connectivity.Connecting: - keys[weight] = struct{}{} - - for j := range p.conns[weight] { - if p.conns[weight][j] == conn { - exists = true - break - } + for j := range p.conns[weight] { + if p.conns[weight][j] == conn { + idx = j + exists = true + break } + } - if !exists { - p.conns[weight] = append(p.conns[weight], conn) - } - - // Problems with connection, try to remove from : - default: - for j := range p.conns[weight] { - if p.conns[weight][j] == conn { - idx = j - exists = true - break - } - } + // if something wrong with connection (bad state or unhealthy), try to close it and remove + if err = isAlive(ctx, p.log, conn); err != nil { + p.log.Warn("connection not alive", + zap.String("address", p.nodes[i].address), + zap.Error(err)) if exists { // remove from connections p.conns[weight] = append(p.conns[weight][:idx], p.conns[weight][idx+1:]...) } - if err := conn.Close(); err != nil { - p.log.Warn("could not close bad connection", zap.Error(err)) + if err = conn.Close(); err != nil { + p.log.Warn("could not close bad connection", + zap.String("address", p.nodes[i].address), + zap.Error(err)) } if p.nodes[i].conn != nil { p.nodes[i].conn = nil } + continue + } + + keys[weight] = struct{}{} + + if !exists { + p.conns[weight] = append(p.conns[weight], conn) } } @@ -223,7 +228,7 @@ func (p *Pool) getConnection(ctx context.Context) (*grpc.ClientConn, error) { p.RLock() defer p.RUnlock() - if p.cur != nil && isAlive(ctx, p.log, p.cur) { + if err := isAlive(ctx, p.log, p.cur); err == nil { return p.cur, nil } @@ -244,7 +249,11 @@ func (p *Pool) getConnection(ctx context.Context) (*grpc.ClientConn, error) { return nil, errNoHealthyConnections } -func isAlive(ctx context.Context, log *zap.Logger, cur *grpc.ClientConn) bool { +func isAlive(ctx context.Context, log *zap.Logger, cur *grpc.ClientConn) error { + if cur == nil { + return errEmptyConnection + } + switch st := cur.GetState(); st { case connectivity.Idle, connectivity.Ready, connectivity.Connecting: req := new(state.HealthRequest) @@ -254,11 +263,13 @@ func isAlive(ctx context.Context, log *zap.Logger, cur *grpc.ClientConn) bool { if err != nil { log.Warn("could not fetch health-check", zap.Error(err)) - return false + return err + } else if !res.Healthy { + return errors.New(res.Status) } - return res.Healthy + return nil default: - return false + return errors.New(st.String()) } } From 307f008dac1b73235697578b74e112679620c767 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 14 Feb 2020 13:06:43 +0300 Subject: [PATCH 020/548] log elapsed time when could not receive object --- receive.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/receive.go b/receive.go index 136156e..c31e89f 100644 --- a/receive.go +++ b/receive.go @@ -7,6 +7,7 @@ import ( "path" "strconv" "strings" + "time" "github.com/labstack/echo/v4" "github.com/nspcc-dev/neofs-api/container" @@ -22,6 +23,7 @@ func (r *router) receiveFile(c echo.Context) error { cid refs.CID oid refs.ObjectID obj *object.Object + start = time.Now() ctx = c.Request().Context() con, err = r.pool.getConnection(ctx) download = c.QueryParam("download") != "" @@ -74,7 +76,9 @@ func (r *router) receiveFile(c echo.Context) error { errors.Wrap(err, "could not prepare connection").Error(), ) } else if obj, err = receiveObject(cli); err != nil { - log.Error("could not receive object", zap.Error(err)) + log.Error("could not receive object", + zap.Duration("elapsed", time.Since(start)), + zap.Error(err)) switch { case strings.Contains(err.Error(), object.ErrNotFound.Error()), From 468cf49126dc540dc6391db605f3ba8eee598794 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 14 Feb 2020 13:07:52 +0300 Subject: [PATCH 021/548] don't panic on http.ErrServerClosed --- main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 4f59a47..f422b36 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "context" "crypto/ecdsa" + "errors" "net/http" _ "net/http/pprof" "time" @@ -63,7 +64,7 @@ func main() { l.Info("run gateway server", zap.String("address", v.GetString("listen_address"))) - if err := e.Start(v.GetString("listen_address")); err != nil { + if err := e.Start(v.GetString("listen_address")); err != nil && !errors.Is(err, http.ErrServerClosed) { l.Panic("could not start server", zap.Error(err)) } }() From c4963f45bdd7e3230434bfc8d2746d4f3488acc9 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 14 Feb 2020 14:35:35 +0300 Subject: [PATCH 022/548] use timer instead of ticker --- main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index f422b36..1e33ea5 100644 --- a/main.go +++ b/main.go @@ -78,7 +78,8 @@ func main() { } func checkConnection(ctx context.Context, p *Pool) { - tick := time.NewTicker(time.Second * 15) + dur := time.Second * 15 + tick := time.NewTimer(dur) loop: for { @@ -87,6 +88,7 @@ loop: break loop case <-tick.C: p.reBalance(ctx) + tick.Reset(dur) } } From 667a98d22fecc768aea5bf816fd647ddcbbf43ff Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 19 Feb 2020 13:45:09 +0300 Subject: [PATCH 023/548] Ignore log files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 690f5d9..8458201 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ cmd/test /plugins/ /vendor/ +*.log testfile .neofs-cli.yml From 65a2da69957b0719baa2566f951068e7dc2856c9 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 19 Feb 2020 13:47:21 +0300 Subject: [PATCH 024/548] Add constants with default values - default values constants - add conn_ttl `GW_CONN_TTL=duration` (gRPC connection time to live) --- settings.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/settings.go b/settings.go index fa024fe..10243da 100644 --- a/settings.go +++ b/settings.go @@ -23,6 +23,16 @@ type empty int const ( devNull = empty(0) generated = "generated" + + minimumTTLInMinutes = 5 + + defaultTTL = minimumTTLInMinutes * time.Minute + + defaultRequestTimeout = 15 * time.Second + defaultConnectTimeout = 30 * time.Second + + defaultKeepaliveTime = 10 * time.Second + defaultKeepaliveTimeout = 10 * time.Second ) func (empty) Read([]byte) (int, error) { return 0, io.EOF } @@ -74,8 +84,9 @@ func settings() *viper.Viper { flags.String("key", generated, `"`+generated+`" to generate key, path to private key file, hex string or wif`) flags.Bool("verbose", false, "debug gRPC connections") - flags.Duration("request_timeout", time.Second*5, "gRPC request timeout") - flags.Duration("connect_timeout", time.Second*30, "gRPC connect timeout") + flags.Duration("request_timeout", defaultRequestTimeout, "gRPC request timeout") + flags.Duration("connect_timeout", defaultConnectTimeout, "gRPC connect timeout") + ttl := flags.DurationP("conn_ttl", "t", defaultTTL, "gRPC connection time to live") flags.String("listen_address", "0.0.0.0:8082", "HTTP Gateway listen address") peers := flags.StringArrayP("peers", "p", nil, "NeoFS nodes") @@ -95,8 +106,9 @@ func settings() *viper.Viper { v.SetDefault("logger.sampling.thereafter", 1000) // keepalive: - v.SetDefault("keepalive.timeout", time.Second*10) - v.SetDefault("keepalive.time", time.Second*10) // If set below 10s, a minimum value of 10s will be used instead. + // If set below 10s, a minimum value of 10s will be used instead. + v.SetDefault("keepalive.time", defaultKeepaliveTime) + v.SetDefault("keepalive.timeout", defaultKeepaliveTimeout) v.SetDefault("keepalive.permit_without_stream", true) if err := v.BindPFlags(flags); err != nil { @@ -119,6 +131,8 @@ func settings() *viper.Viper { case version != nil && *version: fmt.Printf("NeoFS HTTP Gateway %s (%s)\n", Version, Build) os.Exit(0) + case ttl != nil && ttl.Minutes() < minimumTTLInMinutes: + fmt.Printf("connection ttl should not be less than %s", defaultTTL) } if peers != nil && len(*peers) > 0 { From 75113f8195640b443b1707ccdf1f7095fb39855b Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 19 Feb 2020 13:55:42 +0300 Subject: [PATCH 025/548] Refactoring connection pool - replace RWMutex with Mutex - conns now contains `map[int32][]*node` - add index field to node struct - add usedAt field to node struct - `New` don't store current connection twice - writes `used_at` of connection into the log - we check that conn is alive and, that connection used not a long time ago, otherwise, close it - `getConnection` reset current connection and index on errors - `IsAlive` now is a method of `Pool` - `IsAlive` creates context with timeout for every health check call of node --- pool.go | 96 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 67 insertions(+), 29 deletions(-) diff --git a/pool.go b/pool.go index d361881..0395cd4 100644 --- a/pool.go +++ b/pool.go @@ -14,6 +14,7 @@ import ( "github.com/nspcc-dev/neofs-api/service" "github.com/nspcc-dev/neofs-api/state" "github.com/spf13/viper" + "go.uber.org/atomic" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" @@ -22,23 +23,29 @@ import ( type ( node struct { + index int32 address string weight uint32 + usedAt time.Time conn *grpc.ClientConn } Pool struct { log *zap.Logger + ttl time.Duration + connectTimeout time.Duration + requestTimeout time.Duration opts keepalive.ClientParameters - cur *grpc.ClientConn + currentIdx *atomic.Int32 + currentConn *grpc.ClientConn - *sync.RWMutex + *sync.Mutex nodes []*node keys []uint32 - conns map[uint32][]*grpc.ClientConn + conns map[uint32][]*node } ) @@ -49,20 +56,26 @@ var ( func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) *Pool { p := &Pool{ - log: l, - RWMutex: new(sync.RWMutex), - keys: make([]uint32, 0), - nodes: make([]*node, 0), - conns: make(map[uint32][]*grpc.ClientConn), + log: l, + Mutex: new(sync.Mutex), + keys: make([]uint32, 0), + nodes: make([]*node, 0), + conns: make(map[uint32][]*node), + + ttl: defaultTTL, + + currentIdx: atomic.NewInt32(-1), // fill with defaults: - connectTimeout: time.Second * 15, + requestTimeout: defaultRequestTimeout, + connectTimeout: defaultConnectTimeout, opts: keepalive.ClientParameters{ - Time: time.Second * 10, - Timeout: time.Minute * 5, + Time: defaultKeepaliveTime, + Timeout: defaultKeepaliveTimeout, PermitWithoutStream: true, }, } + buf := make([]byte, 8) if _, err := crand.Read(buf); err != nil { l.Panic("could not read seed", zap.Error(err)) @@ -72,6 +85,10 @@ func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) *Pool { rand.Seed(int64(seed)) l.Info("used random seed", zap.Uint64("seed", seed)) + if val := v.GetDuration("conn_ttl"); val > 0 { + p.ttl = val + } + if val := v.GetDuration("connect_timeout"); val > 0 { p.connectTimeout = val } @@ -99,6 +116,7 @@ func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) *Pool { } p.nodes = append(p.nodes, &node{ + index: int32(i), address: address, weight: uint32(weight * 100), }) @@ -110,13 +128,10 @@ func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) *Pool { p.reBalance(ctx) - cur, err := p.getConnection(ctx) - if err != nil { + if _, err := p.getConnection(ctx); err != nil { l.Panic("could get connection", zap.Error(err)) } - p.cur = cur - return p } @@ -173,21 +188,25 @@ func (p *Pool) reBalance(ctx context.Context) { } p.nodes[i].conn = conn + p.nodes[i].usedAt = time.Now() p.log.Info("connected to node", zap.String("address", p.nodes[i].address)) } for j := range p.conns[weight] { - if p.conns[weight][j] == conn { + if p.conns[weight][j] != nil && p.conns[weight][j].conn == conn { idx = j exists = true break } } - // if something wrong with connection (bad state or unhealthy), try to close it and remove - if err = isAlive(ctx, p.log, conn); err != nil { + usedAt := time.Since(p.nodes[i].usedAt) + + // if something wrong with connection (bad state, unhealthy or not used a long time), try to close it and remove + if err = p.isAlive(ctx, conn); err != nil || usedAt > p.ttl { p.log.Warn("connection not alive", zap.String("address", p.nodes[i].address), + zap.Duration("used_at", usedAt), zap.Error(err)) if exists { @@ -198,6 +217,7 @@ func (p *Pool) reBalance(ctx context.Context) { if err = conn.Close(); err != nil { p.log.Warn("could not close bad connection", zap.String("address", p.nodes[i].address), + zap.Duration("used_at", usedAt), zap.Error(err)) } @@ -209,8 +229,12 @@ func (p *Pool) reBalance(ctx context.Context) { keys[weight] = struct{}{} + p.log.Info("connection alive", + zap.String("address", p.nodes[i].address), + zap.Duration("used_at", usedAt)) + if !exists { - p.conns[weight] = append(p.conns[weight], conn) + p.conns[weight] = append(p.conns[weight], p.nodes[i]) } } @@ -225,11 +249,15 @@ func (p *Pool) reBalance(ctx context.Context) { } func (p *Pool) getConnection(ctx context.Context) (*grpc.ClientConn, error) { - p.RLock() - defer p.RUnlock() + p.Lock() + defer p.Unlock() - if err := isAlive(ctx, p.log, p.cur); err == nil { - return p.cur, nil + if err := p.isAlive(ctx, p.currentConn); err == nil { + if id := p.currentIdx.Load(); id != -1 && p.nodes[id] != nil { + p.nodes[id].usedAt = time.Now() + } + + return p.currentConn, nil } for _, w := range p.keys { @@ -237,19 +265,26 @@ func (p *Pool) getConnection(ctx context.Context) (*grpc.ClientConn, error) { case 0: continue case 1: - p.cur = p.conns[w][0] - return p.cur, nil + p.currentConn = p.conns[w][0].conn + p.conns[w][0].usedAt = time.Now() + p.currentIdx.Store(p.conns[w][0].index) + return p.currentConn, nil default: // > 1 i := rand.Intn(ln) - p.cur = p.conns[w][i] - return p.cur, nil + p.currentConn = p.conns[w][i].conn + p.conns[w][i].usedAt = time.Now() + p.currentIdx.Store(p.conns[w][i].index) + return p.currentConn, nil } } + p.currentConn = nil + p.currentIdx.Store(-1) + return nil, errNoHealthyConnections } -func isAlive(ctx context.Context, log *zap.Logger, cur *grpc.ClientConn) error { +func (p *Pool) isAlive(ctx context.Context, cur *grpc.ClientConn) error { if cur == nil { return errEmptyConnection } @@ -259,9 +294,12 @@ func isAlive(ctx context.Context, log *zap.Logger, cur *grpc.ClientConn) error { req := new(state.HealthRequest) req.SetTTL(service.NonForwardingTTL) + ctx, cancel := context.WithTimeout(ctx, p.requestTimeout) + defer cancel() + res, err := state.NewStatusClient(cur).HealthCheck(ctx, req) if err != nil { - log.Warn("could not fetch health-check", zap.Error(err)) + p.log.Warn("could not fetch health-check", zap.Error(err)) return err } else if !res.Healthy { From f86115bf6e58f3aa5aff91f7badfa6d426d00e12 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 19 Feb 2020 15:48:53 +0300 Subject: [PATCH 026/548] go.sum go.uber.org/atomic v1.5.0 --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 528358f..20c3969 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/prometheus/client_golang v1.4.1 // v1.2.1 => v1.4.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.6.2 // v1.6.1 => v1.6.2 + go.uber.org/atomic v1.5.0 go.uber.org/zap v1.13.0 google.golang.org/grpc v1.27.1 // v1.25.1 => v1.27.1 ) From c3a23e91159615be797474e551e24765340752a6 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 20 Feb 2020 17:26:37 +0300 Subject: [PATCH 027/548] debugging gRPC --- go.mod | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 20c3969..a36a09d 100644 --- a/go.mod +++ b/go.mod @@ -17,3 +17,6 @@ require ( // Temporary, before we move repo to github: // replace github.com/nspcc-dev/neofs-proto => ../neofs-proto + +// For debug reasons +replace google.golang.org/grpc => ../grpc-go From 5512bd0b2f62e10207290a21ea360b82e8d87e2e Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 20 Feb 2020 17:36:44 +0300 Subject: [PATCH 028/548] debugging neofs-api --- go.mod | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index a36a09d..a88fae2 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/labstack/echo/v4 v4.1.14 // v4.1.11 => v4.1.14 - github.com/nspcc-dev/neofs-api v0.3.1 // v0.2.14 => v0.3.1 + github.com/nspcc-dev/neofs-api v0.0.0-00000000000000-000000000000 github.com/nspcc-dev/neofs-crypto v0.2.3 github.com/pkg/errors v0.9.1 // v0.8.1 => v0.9.1 github.com/prometheus/client_golang v1.4.1 // v1.2.1 => v1.4.1 @@ -12,7 +12,7 @@ require ( github.com/spf13/viper v1.6.2 // v1.6.1 => v1.6.2 go.uber.org/atomic v1.5.0 go.uber.org/zap v1.13.0 - google.golang.org/grpc v1.27.1 // v1.25.1 => v1.27.1 + google.golang.org/grpc v1.24.0 ) // Temporary, before we move repo to github: @@ -20,3 +20,5 @@ require ( // For debug reasons replace google.golang.org/grpc => ../grpc-go + +replace github.com/nspcc-dev/neofs-api => ../neofs-api From cf61136da5cce02b65b559335fdf5f755c036d30 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 25 Feb 2020 13:31:20 +0300 Subject: [PATCH 029/548] small refactoring --- receive.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/receive.go b/receive.go index c31e89f..7607d00 100644 --- a/receive.go +++ b/receive.go @@ -16,25 +16,23 @@ import ( "github.com/nspcc-dev/neofs-api/service" "github.com/pkg/errors" "go.uber.org/zap" + "google.golang.org/grpc" ) func (r *router) receiveFile(c echo.Context) error { var ( + err error cid refs.CID oid refs.ObjectID obj *object.Object start = time.Now() + con *grpc.ClientConn ctx = c.Request().Context() - con, err = r.pool.getConnection(ctx) download = c.QueryParam("download") != "" ) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, err.Error()) - } - log := r.log.With( - zap.String("node", con.Target()), + // zap.String("node", con.Target()), zap.String("cid", c.Param("cid")), zap.String("oid", c.Param("oid"))) @@ -54,9 +52,20 @@ func (r *router) receiveFile(c echo.Context) error { ) } + { // try to connect or throw http error: + ctx, cancel := context.WithTimeout(ctx, r.timeout) + defer cancel() + + if con, err = r.pool.getConnection(ctx); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + } + ctx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() + log = log.With(zap.String("node", con.Target())) + req := &object.GetRequest{Address: refs.Address{ObjectID: oid, CID: cid}} req.SetTTL(service.SingleForwardingTTL) From 62c6bbd8758fce7653026f9da746ae9ca6ce8804 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 25 Feb 2020 18:35:46 +0300 Subject: [PATCH 030/548] refactoring pool, replace zap.Duration with zap.Stringer --- main.go | 16 +++++++++++++++- pool.go | 26 +++++++++++++++++--------- receive.go | 21 +++++++++++++++------ 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/main.go b/main.go index 1e33ea5..887425d 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,9 @@ type router struct { func main() { var ( + err error + pool *Pool + v = settings() l = newLogger(v) g = newGracefulContext(l) @@ -32,10 +35,21 @@ func main() { grpclog.SetLoggerV2(gRPCLogger(l)) } + switch pool, err = newPool(g, l, v); { + case err == nil: + // ignore + case errors.Is(err, context.Canceled): + l.Info("close application") + return + default: + l.Error("could get connection", zap.Error(err)) + return + } + r := &router{ log: l, + pool: pool, key: fetchKey(l, v), - pool: newPool(g, l, v), timeout: v.GetDuration("request_timeout"), } diff --git a/pool.go b/pool.go index 0395cd4..979fd93 100644 --- a/pool.go +++ b/pool.go @@ -54,7 +54,7 @@ var ( errNoHealthyConnections = errors.New("no active connections") ) -func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) *Pool { +func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) (*Pool, error) { p := &Pool{ log: l, Mutex: new(sync.Mutex), @@ -128,11 +128,9 @@ func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) *Pool { p.reBalance(ctx) - if _, err := p.getConnection(ctx); err != nil { - l.Panic("could get connection", zap.Error(err)) - } + _, err := p.getConnection(ctx) - return p + return p, err } func (p *Pool) close() { @@ -168,6 +166,12 @@ func (p *Pool) reBalance(ctx context.Context) { weight = p.nodes[i].weight ) + if ctx.Err() != nil { + p.log.Warn("something went wrong", zap.Error(ctx.Err())) + + return + } + if conn == nil { p.log.Warn("empty connection, try to connect", zap.String("address", p.nodes[i].address)) @@ -182,7 +186,7 @@ func (p *Pool) reBalance(ctx context.Context) { if err != nil || conn == nil { p.log.Warn("skip, could not connect to node", zap.String("address", p.nodes[i].address), - zap.Duration("elapsed", time.Since(start)), + zap.Stringer("elapsed", time.Since(start)), zap.Error(err)) continue } @@ -206,7 +210,7 @@ func (p *Pool) reBalance(ctx context.Context) { if err = p.isAlive(ctx, conn); err != nil || usedAt > p.ttl { p.log.Warn("connection not alive", zap.String("address", p.nodes[i].address), - zap.Duration("used_at", usedAt), + zap.Stringer("used_at", usedAt), zap.Error(err)) if exists { @@ -217,7 +221,7 @@ func (p *Pool) reBalance(ctx context.Context) { if err = conn.Close(); err != nil { p.log.Warn("could not close bad connection", zap.String("address", p.nodes[i].address), - zap.Duration("used_at", usedAt), + zap.Stringer("used_at", usedAt), zap.Error(err)) } @@ -231,7 +235,7 @@ func (p *Pool) reBalance(ctx context.Context) { p.log.Info("connection alive", zap.String("address", p.nodes[i].address), - zap.Duration("used_at", usedAt)) + zap.Stringer("used_at", usedAt)) if !exists { p.conns[weight] = append(p.conns[weight], p.nodes[i]) @@ -281,6 +285,10 @@ func (p *Pool) getConnection(ctx context.Context) (*grpc.ClientConn, error) { p.currentConn = nil p.currentIdx.Store(-1) + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, errNoHealthyConnections } diff --git a/receive.go b/receive.go index 7607d00..453f23e 100644 --- a/receive.go +++ b/receive.go @@ -28,6 +28,7 @@ func (r *router) receiveFile(c echo.Context) error { start = time.Now() con *grpc.ClientConn ctx = c.Request().Context() + cli object.Service_GetClient download = c.QueryParam("download") != "" ) @@ -36,14 +37,14 @@ func (r *router) receiveFile(c echo.Context) error { zap.String("cid", c.Param("cid")), zap.String("oid", c.Param("oid"))) - if err := cid.Parse(c.Param("cid")); err != nil { + if err = cid.Parse(c.Param("cid")); err != nil { log.Error("wrong container id", zap.Error(err)) return echo.NewHTTPError( http.StatusBadRequest, errors.Wrap(err, "wrong container id").Error(), ) - } else if err := oid.Parse(c.Param("oid")); err != nil { + } else if err = oid.Parse(c.Param("oid")); err != nil { log.Error("wrong object id", zap.Error(err)) return echo.NewHTTPError( @@ -57,6 +58,7 @@ func (r *router) receiveFile(c echo.Context) error { defer cancel() if con, err = r.pool.getConnection(ctx); err != nil { + log.Error("getConnection timeout", zap.Error(err)) return echo.NewHTTPError(http.StatusBadRequest, err.Error()) } } @@ -66,18 +68,25 @@ func (r *router) receiveFile(c echo.Context) error { log = log.With(zap.String("node", con.Target())) + defer func() { + if err != nil { + return + } + + log.Error("object sent to client", zap.Stringer("elapsed", time.Since(start))) + }() + req := &object.GetRequest{Address: refs.Address{ObjectID: oid, CID: cid}} req.SetTTL(service.SingleForwardingTTL) - if err := service.SignRequestHeader(r.key, req); err != nil { + if err = service.SignRequestHeader(r.key, req); err != nil { log.Error("could not sign request", zap.Error(err)) return echo.NewHTTPError( http.StatusBadRequest, errors.Wrap(err, "could not sign request").Error()) } - cli, err := object.NewServiceClient(con).Get(ctx, req) - if err != nil { + if cli, err = object.NewServiceClient(con).Get(ctx, req); err != nil { log.Error("could not prepare connection", zap.Error(err)) return echo.NewHTTPError( @@ -86,7 +95,7 @@ func (r *router) receiveFile(c echo.Context) error { ) } else if obj, err = receiveObject(cli); err != nil { log.Error("could not receive object", - zap.Duration("elapsed", time.Since(start)), + zap.Stringer("elapsed", time.Since(start)), zap.Error(err)) switch { From 4715468f3a4ab1f26a9439ac4751420b42576b52 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 25 Feb 2020 18:38:47 +0300 Subject: [PATCH 031/548] add grpc version to image, add command to build gw with multiple grpc versions --- Makefile | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 5721fe1..0adca33 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ VERSION ?= "$(shell git describe --tags 2>/dev/null | sed 's/^v//')" +GRPC_VERSION=$(shell go list -m google.golang.org/grpc | cut -d " " -f 2) + HUB_IMAGE=nspccdev/neofs B=\033[0;1m @@ -8,7 +10,7 @@ R=\033[0m # Show current version version: - @echo $(VERSION) + @echo $(VERSION)-$(GRPC_VERSION) # Make sure that all files added to commit deps: @@ -19,9 +21,30 @@ deps: @printf "${B}${G}⇒ Store vendor localy${R}: " @go mod vendor && echo OK || (echo fail && exit 2) +image: GRPC_VERSION?= image: deps - @echo "${B}${G}⇒ Build GW docker-image ${R}" + @echo "${B}${G}⇒ Build GW docker-image with $(GRPC_VERSION) ${R}" @docker build \ - --build-arg VERSION=$(VERSION) \ + --build-arg VERSION=$(VERSION)-$(GRPC_VERSION) \ -f Dockerfile \ - -t $(HUB_IMAGE)-http-gate:$(VERSION) . + -t $(HUB_IMAGE)-http-gate:$(VERSION)-$(GRPC_VERSION) . + +.PHONY: dev + +# v1.24.0 v1.25.1 v1.26.0 v1.27.1 +dev: VERSIONS?=$(GRPC_VERSION) +dev: + @echo "=> Build multiple images for $(VERSIONS)"; \ + for v in $(VERSIONS); do \ + curdir=$$(pwd); \ + echo "=> Checkout gRPC to $${v}"; \ + cd ../grpc-go; \ + git checkout $${v} &> /dev/null || (echo "Release $${v} not found" && exit 2); \ + cd ../neofs-api; \ + git checkout go.{sum,mod}; \ + go get google.golang.org/grpc@$${v}; \ + cd $${curdir}; \ + git checkout go.{sum,mod}; \ + go get google.golang.org/grpc@$${v}; \ + make image GRPC_VERSION=$${v}; \ + done \ No newline at end of file From 51574e8274a926bb4618c19581842e2cc7d437b6 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 25 Feb 2020 18:39:35 +0300 Subject: [PATCH 032/548] add more debug information to build --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 99cad69..3777b08 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,16 +6,17 @@ ARG REPO=github.com/nspcc-dev/neofs-gw ENV GOGC off ENV CGO_ENABLED 0 -ENV LDFLAGS "-w -s -X main.Version=${VERSION}" +# add later -w -s +ENV LDFLAGS " -X main.Version=${VERSION} -compressdwarf=false" WORKDIR /src COPY . /src -RUN go build -v -mod=vendor -trimpath -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N)" -o /go/bin/neofs-gw ./ +RUN go build -v -mod=vendor -trimpath -gcflags=all="-N -l" -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N)" -o /go/bin/neofs-gw ./ # Executable image -FROM scratch +FROM alpine WORKDIR / From 2590784e3a9bb375e95569affcd8976631cd6e44 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 25 Feb 2020 18:41:19 +0300 Subject: [PATCH 033/548] add replacements for debug --- go.mod | 9 +++++---- go.sum | 18 ------------------ 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index a88fae2..2b1fad8 100644 --- a/go.mod +++ b/go.mod @@ -12,13 +12,14 @@ require ( github.com/spf13/viper v1.6.2 // v1.6.1 => v1.6.2 go.uber.org/atomic v1.5.0 go.uber.org/zap v1.13.0 - google.golang.org/grpc v1.24.0 + google.golang.org/grpc v1.27.1 ) // Temporary, before we move repo to github: // replace github.com/nspcc-dev/neofs-proto => ../neofs-proto // For debug reasons -replace google.golang.org/grpc => ../grpc-go - -replace github.com/nspcc-dev/neofs-api => ../neofs-api +replace ( + github.com/nspcc-dev/neofs-api => ../neofs-api + google.golang.org/grpc => ../grpc-go +) diff --git a/go.sum b/go.sum index d256e10..2032db3 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -22,7 +21,6 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -124,8 +122,6 @@ github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neofs-api v0.3.1 h1:QbaTcBTUXWjihMR5kw+UPvCjzmLx6g5m3qDvyhyBmzE= -github.com/nspcc-dev/neofs-api v0.3.1/go.mod h1:Y1PQ6nEZyA6sZPDHJxgPWWeT1oU2bKL1N0rpFQ4B/fk= github.com/nspcc-dev/neofs-crypto v0.2.3 h1:aca3X2aly92ENRbFK+kH6Hd+J9EQ4Eu6XMVoITSIKtc= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= @@ -229,14 +225,11 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -249,12 +242,10 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -277,11 +268,9 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= @@ -289,16 +278,10 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -318,6 +301,5 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= From c290e4178a6c2ffcc36e3462672bb3da3d191f86 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 27 Feb 2020 12:51:11 +0300 Subject: [PATCH 034/548] fix typo --- receive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/receive.go b/receive.go index 453f23e..ee1e966 100644 --- a/receive.go +++ b/receive.go @@ -73,7 +73,7 @@ func (r *router) receiveFile(c echo.Context) error { return } - log.Error("object sent to client", zap.Stringer("elapsed", time.Since(start))) + log.Info("object sent to client", zap.Stringer("elapsed", time.Since(start))) }() req := &object.GetRequest{Address: refs.Address{ObjectID: oid, CID: cid}} From 6e917c1982ba5132df4faa5631f1f4d849fc4f56 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 27 Feb 2020 13:05:37 +0300 Subject: [PATCH 035/548] move rebalance timer to settings --- main.go | 6 +++--- settings.go | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 887425d..3974b88 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,7 @@ func main() { v = settings() l = newLogger(v) g = newGracefulContext(l) + d = v.GetDuration("rebalance_timer") ) if v.GetBool("verbose") { @@ -53,7 +54,7 @@ func main() { timeout: v.GetDuration("request_timeout"), } - go checkConnection(g, r.pool) + go checkConnection(g, d, r.pool) e := echo.New() e.Debug = false @@ -91,8 +92,7 @@ func main() { l.Info("stopping server", zap.Error(e.Shutdown(ctx))) } -func checkConnection(ctx context.Context, p *Pool) { - dur := time.Second * 15 +func checkConnection(ctx context.Context, dur time.Duration, p *Pool) { tick := time.NewTimer(dur) loop: diff --git a/settings.go b/settings.go index 10243da..f21bfd9 100644 --- a/settings.go +++ b/settings.go @@ -28,6 +28,7 @@ const ( defaultTTL = minimumTTLInMinutes * time.Minute + defaultRebalanceTimer = 15 * time.Second defaultRequestTimeout = 15 * time.Second defaultConnectTimeout = 30 * time.Second @@ -86,6 +87,8 @@ func settings() *viper.Viper { flags.Bool("verbose", false, "debug gRPC connections") flags.Duration("request_timeout", defaultRequestTimeout, "gRPC request timeout") flags.Duration("connect_timeout", defaultConnectTimeout, "gRPC connect timeout") + flags.Duration("rebalance_timer", defaultRebalanceTimer, "gRPC connection rebalance timer") + ttl := flags.DurationP("conn_ttl", "t", defaultTTL, "gRPC connection time to live") flags.String("listen_address", "0.0.0.0:8082", "HTTP Gateway listen address") From f867fff1d77da115e25d364d0d0efc2f56fbf980 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 27 Feb 2020 13:09:53 +0300 Subject: [PATCH 036/548] ignore test script --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8458201..e957311 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ cmd/test /vendor/ *.log +test.sh testfile .neofs-cli.yml From b8dc1097fbcf09a5fae94e23b4d4e153c2208468 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 19:43:25 +0300 Subject: [PATCH 037/548] update dependencies --- go.mod | 27 ++++++++++------------- go.sum | 67 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 45 insertions(+), 49 deletions(-) diff --git a/go.mod b/go.mod index 2b1fad8..c7bdc19 100644 --- a/go.mod +++ b/go.mod @@ -3,23 +3,18 @@ module github.com/nspcc-dev/neofs-gw go 1.13 require ( - github.com/labstack/echo/v4 v4.1.14 // v4.1.11 => v4.1.14 - github.com/nspcc-dev/neofs-api v0.0.0-00000000000000-000000000000 + github.com/fasthttp/router v0.6.1 + github.com/nspcc-dev/neofs-api v0.4.0 github.com/nspcc-dev/neofs-crypto v0.2.3 - github.com/pkg/errors v0.9.1 // v0.8.1 => v0.9.1 - github.com/prometheus/client_golang v1.4.1 // v1.2.1 => v1.4.1 + github.com/prometheus/client_golang v1.4.1 + github.com/prometheus/common v0.9.1 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.6.2 // v1.6.1 => v1.6.2 - go.uber.org/atomic v1.5.0 - go.uber.org/zap v1.13.0 + github.com/spf13/viper v1.6.2 + github.com/valyala/fasthttp v1.9.0 + go.uber.org/atomic v1.6.0 + go.uber.org/zap v1.14.0 + golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 // indirect + golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect + golang.org/x/text v0.3.2 // indirect google.golang.org/grpc v1.27.1 ) - -// Temporary, before we move repo to github: -// replace github.com/nspcc-dev/neofs-proto => ../neofs-proto - -// For debug reasons -replace ( - github.com/nspcc-dev/neofs-api => ../neofs-api - google.golang.org/grpc => ../grpc-go -) diff --git a/go.sum b/go.sum index 2032db3..d518ce8 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -21,6 +22,7 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -33,6 +35,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fasthttp/router v0.6.1 h1:cPfY4S9tZSh0J62O6h4n6Kxwg9eskQ2GPCNWvXDsa1s= +github.com/fasthttp/router v0.6.1/go.mod h1:00BQmm3xiThNypescxIQ+Gfgw2I/3QWKvuagFoENUb4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -54,7 +58,6 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -84,6 +87,10 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2 h1:Bx0qjetmNjdFXASH02NSAREKpiaDwkO1DRZ3dV2KCcs= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -91,23 +98,10 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/labstack/echo/v4 v4.1.14 h1:h8XP66UfB3tUm+L3QPw7tmwAu3pJaA/nyfHPCcz46ic= -github.com/labstack/echo/v4 v4.1.14/go.mod h1:Q5KZ1vD3V5FEzjM79hjwVrC3ABr7F5IdM23bXQMRDGg= -github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= -github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= @@ -122,13 +116,13 @@ github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= +github.com/nspcc-dev/neofs-api v0.4.0 h1:SwoJ9XV9KxkIc7Nf5WOvMW0XIK6ZxRNXnGVNzp/uQdA= +github.com/nspcc-dev/neofs-api v0.4.0/go.mod h1:Y1PQ6nEZyA6sZPDHJxgPWWeT1oU2bKL1N0rpFQ4B/fk= github.com/nspcc-dev/neofs-crypto v0.2.3 h1:aca3X2aly92ENRbFK+kH6Hd+J9EQ4Eu6XMVoITSIKtc= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= github.com/nspcc-dev/netmap v1.6.1/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= -github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= -github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/tzhash v1.4.0 h1:RVIR+mxOBHl58CE99+DXtE31ylD5PEkZSoWqoj4fVjg= github.com/nspcc-dev/tzhash v1.4.0/go.mod h1:Z8gp/VZbyWgPhaMp/KTmeoW5UTynp/N60g0jTtSzBws= @@ -136,7 +130,6 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -149,7 +142,6 @@ github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFY github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -166,6 +158,8 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f h1:PgA+Olipyj258EIEYnpFFONrrCcAIWNUNoFhUfMqAGY= +github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -199,37 +193,40 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4= -github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw= +github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.14.0 h1:/pduUoebOeeJzTDFuoMgC6nRkiasr1sBCIEorly7m4o= +go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -237,30 +234,27 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -268,9 +262,11 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= @@ -278,14 +274,19 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -296,10 +297,10 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= From 772314d156782514a92e02374136df42a498a2df Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 19:43:39 +0300 Subject: [PATCH 038/548] update dependencies --- go_dev.mod | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 go_dev.mod diff --git a/go_dev.mod b/go_dev.mod new file mode 100644 index 0000000..928773e --- /dev/null +++ b/go_dev.mod @@ -0,0 +1,29 @@ +module github.com/nspcc-dev/neofs-gw + + go 1.13 + + require ( + github.com/fasthttp/router v0.6.1 + github.com/nspcc-dev/neofs-api v0.0.0-00000000000000-000000000000 + github.com/nspcc-dev/neofs-crypto v0.2.3 + github.com/prometheus/client_golang v1.4.1 // v1.2.1 => v1.4.1 + github.com/prometheus/common v0.9.1 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.6.2 // v1.6.1 => v1.6.2 + github.com/valyala/fasthttp v1.9.0 + go.uber.org/atomic v1.6.0 + go.uber.org/zap v1.14.0 + golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 // indirect + golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect + golang.org/x/text v0.3.2 // indirect + google.golang.org/grpc v1.27.1 + ) + + // Temporary, before we move repo to github: + // replace github.com/nspcc-dev/neofs-proto => ../neofs-proto + + // For debug reasons + replace ( + github.com/nspcc-dev/neofs-api => ../neofs-api + google.golang.org/grpc => ../grpc-go + ) From cee4e6239f0aa6ccf0400f6157c1142b5b782ef5 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 19:44:32 +0300 Subject: [PATCH 039/548] add Println and Printf for zapLogger --- logger.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/logger.go b/logger.go index 13aefe0..6c36a8c 100644 --- a/logger.go +++ b/logger.go @@ -114,6 +114,14 @@ func (z *zapLogger) Infof(format string, args ...interface{}) { z.log.Sugar().Infof(format, args...) } +func (z *zapLogger) Println(args ...interface{}) { + z.log.Sugar().Info(args...) +} + +func (z *zapLogger) Printf(format string, args ...interface{}) { + z.log.Sugar().Infof(format, args...) +} + func (z *zapLogger) Warning(args ...interface{}) { z.log.Sugar().Warn(args...) } From 0ab7893337c5fed9a2587ae4395af39cb89ddde3 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 19:45:00 +0300 Subject: [PATCH 040/548] add test environments --- .test.env | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .test.env diff --git a/.test.env b/.test.env new file mode 100644 index 0000000..c3ac45d --- /dev/null +++ b/.test.env @@ -0,0 +1,19 @@ +GW_VERBOSE=--true +GW_KEY=generated + +GW_LISTEN_ADDRESS=0.0.0.0:8087 +GW_LOGGER_LEVEL=debug +GW_CONNECT_TIMEOUT=60s +GW_REQUEST_TIMEOUT=300s +GW_KEEPALIVE_TIMEOUT=300s +GW_KEEPALIVE_TIME=120s +GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=True +GW_CONN_TTL=1h +GW_PEERS_0_WEIGHT=1.0 +GW_PEERS_1_WEIGHT=0.125 +GW_PEERS_2_WEIGHT=0.125 +GW_PEERS_3_WEIGHT=0.125 +GW_PEERS_4_WEIGHT=0.125 +GW_PEERS_5_WEIGHT=0.125 +GW_PEERS_6_WEIGHT=0.125 +GW_PEERS_7_WEIGHT=0.125 \ No newline at end of file From a584012d502ea83e4d92452be2d2006617cb9943 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 19:50:54 +0300 Subject: [PATCH 041/548] update makefile commands: dev, image, version --- Makefile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 0adca33..92c111e 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ R=\033[0m # Show current version version: - @echo $(VERSION)-$(GRPC_VERSION) + @echo "Current version: $(VERSION)-$(GRPC_VERSION)" # Make sure that all files added to commit deps: @@ -21,13 +21,13 @@ deps: @printf "${B}${G}⇒ Store vendor localy${R}: " @go mod vendor && echo OK || (echo fail && exit 2) -image: GRPC_VERSION?= +image: VERSION?= image: deps @echo "${B}${G}⇒ Build GW docker-image with $(GRPC_VERSION) ${R}" @docker build \ - --build-arg VERSION=$(VERSION)-$(GRPC_VERSION) \ + --build-arg VERSION=$(VERSION) \ -f Dockerfile \ - -t $(HUB_IMAGE)-http-gate:$(VERSION)-$(GRPC_VERSION) . + -t $(HUB_IMAGE)-http-gate:$(VERSION) . .PHONY: dev @@ -45,6 +45,7 @@ dev: go get google.golang.org/grpc@$${v}; \ cd $${curdir}; \ git checkout go.{sum,mod}; \ + cp go_dev.mod go.sum; \ go get google.golang.org/grpc@$${v}; \ - make image GRPC_VERSION=$${v}; \ + make image VERSION=$(VERSION)-$${v}; \ done \ No newline at end of file From dd213b0c416255efe57e4a3a7fb9405ae16ce4c5 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 19:51:49 +0300 Subject: [PATCH 042/548] update makefile commands: dev, image, version --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 92c111e..92cb5fc 100644 --- a/Makefile +++ b/Makefile @@ -44,8 +44,8 @@ dev: git checkout go.{sum,mod}; \ go get google.golang.org/grpc@$${v}; \ cd $${curdir}; \ - git checkout go.{sum,mod}; \ cp go_dev.mod go.sum; \ go get google.golang.org/grpc@$${v}; \ make image VERSION=$(VERSION)-$${v}; \ + git checkout go.{sum,mod}; \ done \ No newline at end of file From d6df78974e83192872ebbc1fdef9607c57d004c7 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 19:52:33 +0300 Subject: [PATCH 043/548] ignore blast benchmark config --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e957311..104aa0a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ cmd/test *.log test.sh testfile +.blast.yml .neofs-cli.yml From c6a06ca7098f011bc1802da246336f5ddcedc98a Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 19:56:22 +0300 Subject: [PATCH 044/548] add fasthttp metrics handler --- metrics.go | 127 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 metrics.go diff --git a/metrics.go b/metrics.go new file mode 100644 index 0000000..7a4b51d --- /dev/null +++ b/metrics.go @@ -0,0 +1,127 @@ +package main + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/expfmt" + "github.com/valyala/fasthttp" +) + +func metricsHandler(reg prometheus.Gatherer, opts promhttp.HandlerOpts) fasthttp.RequestHandler { + var ( + inFlightSem chan struct{} + errCnt = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "promhttp_metric_handler_errors_total", + Help: "Total number of internal errors encountered by the promhttp metric handler.", + }, + []string{"cause"}, + ) + ) + + if opts.MaxRequestsInFlight > 0 { + inFlightSem = make(chan struct{}, opts.MaxRequestsInFlight) + } + if opts.Registry != nil { + // Initialize all possibilites that can occur below. + errCnt.WithLabelValues("gathering") + errCnt.WithLabelValues("encoding") + if err := opts.Registry.Register(errCnt); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + errCnt = are.ExistingCollector.(*prometheus.CounterVec) + } else { + panic(err) + } + } + } + + h := fasthttp.RequestHandler(func(c *fasthttp.RequestCtx) { + if inFlightSem != nil { + select { + case inFlightSem <- struct{}{}: // All good, carry on. + defer func() { <-inFlightSem }() + default: + + c.Error(fmt.Sprintf( + "Limit of concurrent requests reached (%d), try again later.", opts.MaxRequestsInFlight, + ), fasthttp.StatusServiceUnavailable) + return + } + } + mfs, err := reg.Gather() + if err != nil { + if opts.ErrorLog != nil { + panic("error gathering metrics:" + err.Error()) + } + + errCnt.WithLabelValues("gathering").Inc() + switch opts.ErrorHandling { + case promhttp.PanicOnError: + panic(err) + case promhttp.ContinueOnError: + if len(mfs) == 0 { + // Still report the error if no metrics have been gathered. + c.Error(err.Error(), fasthttp.StatusServiceUnavailable) + return + } + case promhttp.HTTPErrorOnError: + c.Error(err.Error(), fasthttp.StatusServiceUnavailable) + return + } + } + + contentType := expfmt.FmtText + c.SetContentType(string(contentType)) + enc := expfmt.NewEncoder(c, contentType) + + var lastErr error + + // handleError handles the error according to opts.ErrorHandling + // and returns true if we have to abort after the handling. + handleError := func(err error) bool { + if err == nil { + return false + } + lastErr = err + if opts.ErrorLog != nil { + opts.ErrorLog.Println("error encoding and sending metric family:", err) + } + errCnt.WithLabelValues("encoding").Inc() + switch opts.ErrorHandling { + case promhttp.PanicOnError: + panic(err) + case promhttp.HTTPErrorOnError: + c.Error(err.Error(), fasthttp.StatusServiceUnavailable) + return true + } + // Do nothing in all other cases, including ContinueOnError. + return false + } + + for _, mf := range mfs { + fmt.Println(*mf.Name) + if handleError(enc.Encode(mf)) { + return + } + } + if closer, ok := enc.(expfmt.Closer); ok { + // This in particular takes care of the final "# EOF\n" line for OpenMetrics. + if handleError(closer.Close()) { + return + } + } + + handleError(lastErr) + }) + + if opts.Timeout <= 0 { + return h + } + + return fasthttp.TimeoutHandler(h, opts.Timeout, fmt.Sprintf( + "Exceeded configured timeout of %v.\n", + opts.Timeout, + )) +} From 437133e280a2f2cd7b5bc6ccfff3ed5283fd5f4b Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 19:56:50 +0300 Subject: [PATCH 045/548] add fasthttp pprof handler --- pprof.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 pprof.go diff --git a/pprof.go b/pprof.go new file mode 100644 index 0000000..97b6424 --- /dev/null +++ b/pprof.go @@ -0,0 +1,37 @@ +package main + +import ( + "net/http/pprof" + rtp "runtime/pprof" + + "github.com/valyala/fasthttp" + "github.com/valyala/fasthttp/fasthttpadaptor" +) + +func pprofHandler() fasthttp.RequestHandler { + items := rtp.Profiles() + + profiles := map[string]fasthttp.RequestHandler{ + "": fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Index), + "cmdline": fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Cmdline), + "profile": fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Profile), + "symbol": fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Symbol), + "trace": fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Trace), + } + + for i := range items { + name := items[i].Name() + profiles[name] = fasthttpadaptor.NewFastHTTPHandler(pprof.Handler(name)) + } + + return func(ctx *fasthttp.RequestCtx) { + name, _ := ctx.UserValue("name").(string) + + if handler, ok := profiles[name]; ok { + handler(ctx) + return + } + + ctx.Error("Not found", fasthttp.StatusNotFound) + } +} From 35e2378afca2c1a5e00c5e6d66a073aeb6d1766f Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 20:03:56 +0300 Subject: [PATCH 046/548] refactoring receive file handler: use fasthttp instead echo --- receive.go | 153 ++++++++++++++++++++++++++++------------------------- 1 file changed, 81 insertions(+), 72 deletions(-) diff --git a/receive.go b/receive.go index ee1e966..9b89cad 100644 --- a/receive.go +++ b/receive.go @@ -9,61 +9,57 @@ import ( "strings" "time" - "github.com/labstack/echo/v4" "github.com/nspcc-dev/neofs-api/container" "github.com/nspcc-dev/neofs-api/object" "github.com/nspcc-dev/neofs-api/refs" "github.com/nspcc-dev/neofs-api/service" - "github.com/pkg/errors" + "github.com/valyala/fasthttp" "go.uber.org/zap" "google.golang.org/grpc" ) -func (r *router) receiveFile(c echo.Context) error { +func (a *app) receiveFile(c *fasthttp.RequestCtx) { var ( - err error - cid refs.CID - oid refs.ObjectID - obj *object.Object - start = time.Now() - con *grpc.ClientConn - ctx = c.Request().Context() - cli object.Service_GetClient - download = c.QueryParam("download") != "" + err error + cid refs.CID + oid refs.ObjectID + start = time.Now() + con *grpc.ClientConn + cli object.Service_GetClient + ctx context.Context + sCID, _ = c.UserValue("cid").(string) + sOID, _ = c.UserValue("oid").(string) ) - log := r.log.With( + log := a.log.With( // zap.String("node", con.Target()), - zap.String("cid", c.Param("cid")), - zap.String("oid", c.Param("oid"))) + zap.String("cid", sCID), + zap.String("oid", sOID)) - if err = cid.Parse(c.Param("cid")); err != nil { + if err = cid.Parse(sCID); err != nil { log.Error("wrong container id", zap.Error(err)) - return echo.NewHTTPError( - http.StatusBadRequest, - errors.Wrap(err, "wrong container id").Error(), - ) - } else if err = oid.Parse(c.Param("oid")); err != nil { + c.Error("wrong container id", fasthttp.StatusBadRequest) + return + } else if err = oid.Parse(sOID); err != nil { log.Error("wrong object id", zap.Error(err)) - return echo.NewHTTPError( - http.StatusBadRequest, - errors.Wrap(err, "wrong object id").Error(), - ) + c.Error("wrong object id", fasthttp.StatusBadRequest) + return } { // try to connect or throw http error: - ctx, cancel := context.WithTimeout(ctx, r.timeout) + ctx, cancel := context.WithTimeout(c, a.timeout) defer cancel() - if con, err = r.pool.getConnection(ctx); err != nil { + if con, err = a.pool.getConnection(ctx); err != nil { log.Error("getConnection timeout", zap.Error(err)) - return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + c.Error("could not get alive connection", fasthttp.StatusBadRequest) + return } } - ctx, cancel := context.WithTimeout(ctx, r.timeout) + ctx, cancel := context.WithTimeout(c, a.timeout) defer cancel() log = log.With(zap.String("node", con.Target())) @@ -79,21 +75,18 @@ func (r *router) receiveFile(c echo.Context) error { req := &object.GetRequest{Address: refs.Address{ObjectID: oid, CID: cid}} req.SetTTL(service.SingleForwardingTTL) - if err = service.SignRequestHeader(r.key, req); err != nil { + if err = service.SignRequestHeader(a.key, req); err != nil { log.Error("could not sign request", zap.Error(err)) - return echo.NewHTTPError( - http.StatusBadRequest, - errors.Wrap(err, "could not sign request").Error()) + c.Error("could not sign request", fasthttp.StatusBadRequest) + return } if cli, err = object.NewServiceClient(con).Get(ctx, req); err != nil { log.Error("could not prepare connection", zap.Error(err)) - return echo.NewHTTPError( - http.StatusBadRequest, - errors.Wrap(err, "could not prepare connection").Error(), - ) - } else if obj, err = receiveObject(cli); err != nil { + c.Error("could not prepare connection", fasthttp.StatusBadRequest) + return + } else if err = receiveObject(c, cli); err != nil { log.Error("could not receive object", zap.Stringer("elapsed", time.Since(start)), zap.Error(err)) @@ -101,52 +94,68 @@ func (r *router) receiveFile(c echo.Context) error { switch { case strings.Contains(err.Error(), object.ErrNotFound.Error()), strings.Contains(err.Error(), container.ErrNotFound.Error()): - return echo.NewHTTPError(http.StatusNotFound, err.Error()) + c.Error("object not found", fasthttp.StatusNotFound) default: - return echo.NewHTTPError( - http.StatusBadRequest, - errors.Wrap(err, "could not receive object").Error(), - ) + c.Error("could not receive object", fasthttp.StatusBadRequest) } + + return } - - log.Info("object fetched successfully") - - c.Response().Header().Set("Content-Length", strconv.FormatUint(obj.SystemHeader.PayloadLength, 10)) - c.Response().Header().Set("x-object-id", obj.SystemHeader.ID.String()) - c.Response().Header().Set("x-owner-id", obj.SystemHeader.OwnerID.String()) - c.Response().Header().Set("x-container-id", obj.SystemHeader.CID.String()) - - for i := range obj.Headers { - if hdr := obj.Headers[i].GetUserHeader(); hdr != nil { - c.Response().Header().Set("x-"+hdr.Key, hdr.Value) - - if hdr.Key == object.FilenameHeader && download { - // NOTE: we use path.Base because hdr.Value can be something like `/path/to/filename.ext` - c.Response().Header().Set("Content-Disposition", "attachment; filename="+path.Base(hdr.Value)) - } - } - } - - return c.Blob(http.StatusOK, - http.DetectContentType(obj.Payload), - obj.Payload) } -func receiveObject(cli object.Service_GetClient) (*object.Object, error) { - var obj *object.Object +func receiveObject(c *fasthttp.RequestCtx, cli object.Service_GetClient) error { + var ( + typ string + put = c.Request.URI().QueryArgs().GetBool("download") + ) + for { resp, err := cli.Recv() if err != nil { if err == io.EOF { break } - return nil, err - } else if obj == nil { - obj = resp.GetObject() + return err } - obj.Payload = append(obj.Payload, resp.GetChunk()...) + switch o := resp.R.(type) { + case *object.GetResponse_Object: + obj := o.Object + + c.Response.Header.Set("Content-Length", strconv.FormatUint(obj.SystemHeader.PayloadLength, 10)) + c.Response.Header.Set("x-object-id", obj.SystemHeader.ID.String()) + c.Response.Header.Set("x-owner-id", obj.SystemHeader.OwnerID.String()) + c.Response.Header.Set("x-container-id", obj.SystemHeader.CID.String()) + + for i := range obj.Headers { + if hdr := obj.Headers[i].GetUserHeader(); hdr != nil { + c.Response.Header.Set("x-"+hdr.Key, hdr.Value) + + if hdr.Key == object.FilenameHeader && put { + // NOTE: we use path.Base because hdr.Value can be something like `/path/to/filename.ext` + c.Response.Header.Set("Content-Disposition", "attachment; filename="+path.Base(hdr.Value)) + } + } + } + + typ = http.DetectContentType(obj.Payload) + + if _, err = c.Write(obj.Payload); err != nil { + return err + } + + case *object.GetResponse_Chunk: + if typ == "" { + typ = http.DetectContentType(o.Chunk) + } + + if _, err = c.Write(o.Chunk); err != nil { + return err + } + } } - return obj, nil + + c.SetContentType(typ) + + return nil } From 7790684726e4ee5f19d4b68487f138d8e7abff6c Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 20:05:25 +0300 Subject: [PATCH 047/548] refactoring connectionpool: add healthcheck, move getConnection and first call reBalance outside constructor --- pool.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pool.go b/pool.go index 979fd93..7baa273 100644 --- a/pool.go +++ b/pool.go @@ -46,15 +46,18 @@ type ( nodes []*node keys []uint32 conns map[uint32][]*node + + unhealthy *atomic.Error } ) var ( + errBootstrapping = errors.New("bootstrapping") errEmptyConnection = errors.New("empty connection") errNoHealthyConnections = errors.New("no active connections") ) -func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) (*Pool, error) { +func newPool(l *zap.Logger, v *viper.Viper) *Pool { p := &Pool{ log: l, Mutex: new(sync.Mutex), @@ -74,6 +77,8 @@ func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) (*Pool, error) Timeout: defaultKeepaliveTimeout, PermitWithoutStream: true, }, + + unhealthy: atomic.NewError(errBootstrapping), } buf := make([]byte, 8) @@ -126,11 +131,7 @@ func newPool(ctx context.Context, l *zap.Logger, v *viper.Viper) (*Pool, error) zap.Uint32("weight", p.nodes[i].weight)) } - p.reBalance(ctx) - - _, err := p.getConnection(ctx) - - return p, err + return p } func (p *Pool) close() { @@ -150,7 +151,12 @@ func (p *Pool) close() { func (p *Pool) reBalance(ctx context.Context) { p.Lock() - defer p.Unlock() + defer func() { + p.Unlock() + + _, err := p.getConnection(ctx) + p.unhealthy.Store(err) + }() keys := make(map[uint32]struct{}) @@ -166,8 +172,9 @@ func (p *Pool) reBalance(ctx context.Context) { weight = p.nodes[i].weight ) - if ctx.Err() != nil { - p.log.Warn("something went wrong", zap.Error(ctx.Err())) + if err = ctx.Err(); err != nil { + p.log.Warn("something went wrong", zap.Error(err)) + p.unhealthy.Store(err) return } From 8aef59cdbd49b2cabf5a5d0d1322d7c96673936e Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 20:07:50 +0300 Subject: [PATCH 048/548] refactoring application: use fasthttp instead echo, add healthcheck handlers, run web server immidiently, close pool connections when stop --- main.go | 115 ++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 79 insertions(+), 36 deletions(-) diff --git a/main.go b/main.go index 3974b88..ad17d6e 100644 --- a/main.go +++ b/main.go @@ -4,92 +4,134 @@ import ( "context" "crypto/ecdsa" "errors" - "net/http" - _ "net/http/pprof" "time" - "github.com/labstack/echo/v4" - "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/fasthttp/router" + "github.com/prometheus/client_golang/prometheus" + http "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/valyala/fasthttp" "go.uber.org/zap" "google.golang.org/grpc/grpclog" ) -type router struct { +type app struct { pool *Pool log *zap.Logger timeout time.Duration key *ecdsa.PrivateKey } +const ( + defaultHealthyMsg = "NeoFS HTTP Gateway is " + defaultContentType = "text/plain; charset=utf-8" +) + func main() { var ( - err error - pool *Pool + err error v = settings() l = newLogger(v) + z = gRPCLogger(l) g = newGracefulContext(l) d = v.GetDuration("rebalance_timer") ) if v.GetBool("verbose") { - grpclog.SetLoggerV2(gRPCLogger(l)) + grpclog.SetLoggerV2(z) } - switch pool, err = newPool(g, l, v); { - case err == nil: - // ignore - case errors.Is(err, context.Canceled): - l.Info("close application") - return - default: - l.Error("could get connection", zap.Error(err)) - return - } - - r := &router{ + a := &app{ log: l, - pool: pool, + pool: newPool(l, v), key: fetchKey(l, v), timeout: v.GetDuration("request_timeout"), } - go checkConnection(g, d, r.pool) + r := router.New() + r.RedirectTrailingSlash = true + r.GET("/get/:cid/:oid", a.receiveFile) - e := echo.New() - e.Debug = false - e.HidePort = true - e.HideBanner = true + r.GET("/-/ready", func(ctx *fasthttp.RequestCtx) { + ctx.SetStatusCode(fasthttp.StatusOK) + ctx.SetBodyString("NeoFS HTTP Gateway is ready") + }) - e.GET("/:cid/:oid", r.receiveFile) + r.GET("/-/healthy", func(c *fasthttp.RequestCtx) { + code := fasthttp.StatusOK + msg := "healthy" + + if err := a.pool.unhealthy.Load(); err != nil { + msg = "unhealthy: " + err.Error() + code = fasthttp.StatusBadRequest + } + + c.Response.Reset() + c.SetStatusCode(code) + c.SetContentType(defaultContentType) + c.SetBodyString(defaultHealthyMsg + msg) + }) // enable metrics if v.GetBool("metrics") { l.Info("enabled /metrics") - e.GET("/metrics", echo.WrapHandler(promhttp.Handler())) + r.GET("/metrics/", metricsHandler(prometheus.DefaultGatherer, http.HandlerOpts{ + ErrorLog: z.(http.Logger), + //ErrorHandling: 0, + //Registry: nil, + //DisableCompression: false, + //MaxRequestsInFlight: 0, + //Timeout: 0, + //EnableOpenMetrics: false, + })) } // enable pprof if v.GetBool("pprof") { l.Info("enabled /debug/pprof") - e.Any("/debug/pprof*", echo.WrapHandler(http.DefaultServeMux)) + r.GET("/debug/pprof/", pprofHandler()) + r.GET("/debug/pprof/:name", pprofHandler()) + } + + en := &fasthttp.Server{ + Name: "neofs-http-gate", + Handler: r.Handler, + ReadBufferSize: 4096, + ReadTimeout: time.Second * 15, + GetOnly: true, + DisableHeaderNamesNormalizing: true, + NoDefaultServerHeader: true, + NoDefaultContentType: true, } go func() { + bind := v.GetString("listen_address") l.Info("run gateway server", - zap.String("address", v.GetString("listen_address"))) + zap.String("address", bind)) - if err := e.Start(v.GetString("listen_address")); err != nil && !errors.Is(err, http.ErrServerClosed) { + if err := en.ListenAndServe(bind); err != nil { l.Panic("could not start server", zap.Error(err)) } }() + go checkConnection(g, d, a.pool) + + a.pool.reBalance(g) + + switch _, err = a.pool.getConnection(g); { + case err == nil: + // ignore + case errors.Is(err, context.Canceled): + // ignore + // l.Info("context canceled") + default: + l.Error("could get connection", zap.Error(err)) + return + } + <-g.Done() - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*30) - defer cancel() - - l.Info("stopping server", zap.Error(e.Shutdown(ctx))) + l.Info("web server stopped", zap.Error(en.Shutdown())) } func checkConnection(ctx context.Context, dur time.Duration, p *Pool) { @@ -106,7 +148,8 @@ loop: } } + p.close() tick.Stop() - p.log.Info("stop connection worker") + p.log.Info("connection worker stopped") } From cf0f9ce5f6d3123ff9f6f9604ff94ff9749f3c7d Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 28 Feb 2020 20:11:20 +0300 Subject: [PATCH 049/548] fixes for dev version of go.mod --- go_dev.mod | 47 ++++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/go_dev.mod b/go_dev.mod index 928773e..aaf657b 100644 --- a/go_dev.mod +++ b/go_dev.mod @@ -1,29 +1,26 @@ module github.com/nspcc-dev/neofs-gw - go 1.13 +go 1.13 - require ( - github.com/fasthttp/router v0.6.1 - github.com/nspcc-dev/neofs-api v0.0.0-00000000000000-000000000000 - github.com/nspcc-dev/neofs-crypto v0.2.3 - github.com/prometheus/client_golang v1.4.1 // v1.2.1 => v1.4.1 - github.com/prometheus/common v0.9.1 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.6.2 // v1.6.1 => v1.6.2 - github.com/valyala/fasthttp v1.9.0 - go.uber.org/atomic v1.6.0 - go.uber.org/zap v1.14.0 - golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 // indirect - golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect - golang.org/x/text v0.3.2 // indirect - google.golang.org/grpc v1.27.1 - ) +require ( + github.com/fasthttp/router v0.6.1 + github.com/nspcc-dev/neofs-api v0.0.0-00000000000000-000000000000 + github.com/nspcc-dev/neofs-crypto v0.2.3 + github.com/prometheus/client_golang v1.4.1 // v1.2.1 => v1.4.1 + github.com/prometheus/common v0.9.1 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.6.2 // v1.6.1 => v1.6.2 + github.com/valyala/fasthttp v1.9.0 + go.uber.org/atomic v1.6.0 + go.uber.org/zap v1.14.0 + golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 // indirect + golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect + golang.org/x/text v0.3.2 // indirect + google.golang.org/grpc v1.27.1 +) - // Temporary, before we move repo to github: - // replace github.com/nspcc-dev/neofs-proto => ../neofs-proto - - // For debug reasons - replace ( - github.com/nspcc-dev/neofs-api => ../neofs-api - google.golang.org/grpc => ../grpc-go - ) +// For debug reasons +replace ( + github.com/nspcc-dev/neofs-api => ../neofs-api + google.golang.org/grpc => ../grpc-go +) From 758006dea03b52c4d5671383799f4402fd4a1a82 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Sat, 29 Feb 2020 13:25:22 +0300 Subject: [PATCH 050/548] update makefile dev command --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 92cb5fc..db412f2 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,7 @@ image: deps dev: VERSIONS?=$(GRPC_VERSION) dev: @echo "=> Build multiple images for $(VERSIONS)"; \ + git checkout go.{sum,mod}; \ for v in $(VERSIONS); do \ curdir=$$(pwd); \ echo "=> Checkout gRPC to $${v}"; \ @@ -44,7 +45,7 @@ dev: git checkout go.{sum,mod}; \ go get google.golang.org/grpc@$${v}; \ cd $${curdir}; \ - cp go_dev.mod go.sum; \ + cp go_dev.mod go.mod; \ go get google.golang.org/grpc@$${v}; \ make image VERSION=$(VERSION)-$${v}; \ git checkout go.{sum,mod}; \ From 131d99a182b284179308bc28d6050bb7fe194fe8 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Sat, 29 Feb 2020 16:40:54 +0300 Subject: [PATCH 051/548] fix content-type detection --- receive.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/receive.go b/receive.go index 9b89cad..a3d001b 100644 --- a/receive.go +++ b/receive.go @@ -138,7 +138,9 @@ func receiveObject(c *fasthttp.RequestCtx, cli object.Service_GetClient) error { } } - typ = http.DetectContentType(obj.Payload) + if len(obj.Payload) > 0 { + typ = http.DetectContentType(obj.Payload) + } if _, err = c.Write(obj.Payload); err != nil { return err From 5a640fc0fa3658201355b32f54438ce9aef71a6b Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Sat, 29 Feb 2020 17:07:07 +0300 Subject: [PATCH 052/548] remove debug information --- metrics.go | 1 - 1 file changed, 1 deletion(-) diff --git a/metrics.go b/metrics.go index 7a4b51d..4829d35 100644 --- a/metrics.go +++ b/metrics.go @@ -101,7 +101,6 @@ func metricsHandler(reg prometheus.Gatherer, opts promhttp.HandlerOpts) fasthttp } for _, mf := range mfs { - fmt.Println(*mf.Name) if handleError(enc.Encode(mf)) { return } From dba58f3935cbf88d7b1999f3a391c9f32b1429db Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Sat, 29 Feb 2020 18:00:20 +0300 Subject: [PATCH 053/548] refactored logger wrapper --- logger.go | 69 ++++++++++++++++++------------------------------------- 1 file changed, 22 insertions(+), 47 deletions(-) diff --git a/logger.go b/logger.go index 6c36a8c..21588e1 100644 --- a/logger.go +++ b/logger.go @@ -11,7 +11,8 @@ import ( ) type zapLogger struct { - log *zap.Logger + zapcore.Core + log *zap.SugaredLogger } const ( @@ -23,10 +24,12 @@ const ( ) func gRPCLogger(l *zap.Logger) grpclog.LoggerV2 { + log := l.WithOptions( + // skip gRPCLog + zapLogger in caller + zap.AddCallerSkip(2)) return &zapLogger{ - log: l.WithOptions( - // skip gRPCLog + zapLogger in caller - zap.AddCallerSkip(2)), + Core: log.Core(), + log: log.Sugar(), } } @@ -102,60 +105,32 @@ func newLogger(v *viper.Viper) *zap.Logger { zap.String("app_version", version)) } -func (z *zapLogger) Info(args ...interface{}) { - z.log.Sugar().Info(args...) -} +func (z *zapLogger) Info(args ...interface{}) { z.log.Info(args...) } -func (z *zapLogger) Infoln(args ...interface{}) { - z.log.Sugar().Info(args...) -} +func (z *zapLogger) Infoln(args ...interface{}) { z.log.Info(args...) } -func (z *zapLogger) Infof(format string, args ...interface{}) { - z.log.Sugar().Infof(format, args...) -} +func (z *zapLogger) Infof(format string, args ...interface{}) { z.log.Infof(format, args...) } -func (z *zapLogger) Println(args ...interface{}) { - z.log.Sugar().Info(args...) -} +func (z *zapLogger) Println(args ...interface{}) { z.log.Info(args...) } -func (z *zapLogger) Printf(format string, args ...interface{}) { - z.log.Sugar().Infof(format, args...) -} +func (z *zapLogger) Printf(format string, args ...interface{}) { z.log.Infof(format, args...) } -func (z *zapLogger) Warning(args ...interface{}) { - z.log.Sugar().Warn(args...) -} +func (z *zapLogger) Warning(args ...interface{}) { z.log.Warn(args...) } -func (z *zapLogger) Warningln(args ...interface{}) { - z.log.Sugar().Warn(args...) -} +func (z *zapLogger) Warningln(args ...interface{}) { z.log.Warn(args...) } -func (z *zapLogger) Warningf(format string, args ...interface{}) { - z.log.Sugar().Warnf(format, args...) -} +func (z *zapLogger) Warningf(format string, args ...interface{}) { z.log.Warnf(format, args...) } -func (z *zapLogger) Error(args ...interface{}) { - z.log.Sugar().Error(args...) -} +func (z *zapLogger) Error(args ...interface{}) { z.log.Error(args...) } -func (z *zapLogger) Errorln(args ...interface{}) { - z.log.Sugar().Error(args...) -} +func (z *zapLogger) Errorln(args ...interface{}) { z.log.Error(args...) } -func (z *zapLogger) Errorf(format string, args ...interface{}) { - z.log.Sugar().Errorf(format, args...) -} +func (z *zapLogger) Errorf(format string, args ...interface{}) { z.log.Errorf(format, args...) } -func (z *zapLogger) Fatal(args ...interface{}) { - z.log.Sugar().Fatal(args...) -} +func (z *zapLogger) Fatal(args ...interface{}) { z.log.Fatal(args...) } -func (z *zapLogger) Fatalln(args ...interface{}) { - z.log.Sugar().Fatal(args...) -} +func (z *zapLogger) Fatalln(args ...interface{}) { z.log.Fatal(args...) } -func (z *zapLogger) Fatalf(format string, args ...interface{}) { - z.log.Sugar().Fatalf(format, args...) -} +func (z *zapLogger) Fatalf(format string, args ...interface{}) { z.Fatalf(format, args...) } -func (z *zapLogger) V(int) bool { return z.log.Core().Enabled(zapcore.DebugLevel) } +func (z *zapLogger) V(int) bool { return z.Enabled(zapcore.DebugLevel) } From 1d5c1897a372b973235a18459ae3b1f9a5144989 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 3 Mar 2020 13:34:26 +0300 Subject: [PATCH 054/548] reusable health checkers endpoints --- health.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 health.go diff --git a/health.go b/health.go new file mode 100644 index 0000000..708bbb7 --- /dev/null +++ b/health.go @@ -0,0 +1,34 @@ +package main + +import ( + "github.com/fasthttp/router" + "github.com/valyala/fasthttp" + "go.uber.org/atomic" +) + +const ( + healthyState = "NeoFS HTTP Gateway is " + defaultContentType = "text/plain; charset=utf-8" +) + +func attachHealthy(r *router.Router, e *atomic.Error) { + r.GET("/-/ready", func(ctx *fasthttp.RequestCtx) { + ctx.SetStatusCode(fasthttp.StatusOK) + ctx.SetBodyString(healthyState + "ready") + }) + + r.GET("/-/healthy", func(c *fasthttp.RequestCtx) { + code := fasthttp.StatusOK + msg := "healthy" + + if err := e.Load(); err != nil { + msg = "unhealthy: " + err.Error() + code = fasthttp.StatusBadRequest + } + + c.Response.Reset() + c.SetStatusCode(code) + c.SetContentType(defaultContentType) + c.SetBodyString(healthyState + msg) + }) +} From ad03a95feb51d797c95e6ec53420347b66a42f14 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 3 Mar 2020 13:34:52 +0300 Subject: [PATCH 055/548] added reusable metrics endpoints --- metrics.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/metrics.go b/metrics.go index 4829d35..4a7c857 100644 --- a/metrics.go +++ b/metrics.go @@ -3,12 +3,25 @@ package main import ( "fmt" + "github.com/fasthttp/router" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/expfmt" "github.com/valyala/fasthttp" ) +func attachMetrics(r *router.Router, z promhttp.Logger) { + r.GET("/metrics/", metricsHandler(prometheus.DefaultGatherer, promhttp.HandlerOpts{ + ErrorLog: z, + //ErrorHandling: 0, + //Registry: nil, + //DisableCompression: false, + //MaxRequestsInFlight: 0, + //Timeout: 0, + //EnableOpenMetrics: false, + })) +} + func metricsHandler(reg prometheus.Gatherer, opts promhttp.HandlerOpts) fasthttp.RequestHandler { var ( inFlightSem chan struct{} From 50e74ee3a1053ff4f42e9d0fc85a9787ee74d899 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 3 Mar 2020 13:35:06 +0300 Subject: [PATCH 056/548] added reusable pprof endpoints --- pprof.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pprof.go b/pprof.go index 97b6424..ef5457a 100644 --- a/pprof.go +++ b/pprof.go @@ -4,10 +4,16 @@ import ( "net/http/pprof" rtp "runtime/pprof" + "github.com/fasthttp/router" "github.com/valyala/fasthttp" "github.com/valyala/fasthttp/fasthttpadaptor" ) +func attachProfiler(r *router.Router) { + r.GET("/debug/pprof/", pprofHandler()) + r.GET("/debug/pprof/:name", pprofHandler()) +} + func pprofHandler() fasthttp.RequestHandler { items := rtp.Profiles() From 2db83caf29630b5a65604d1b50a1df946bf92abd Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 3 Mar 2020 13:36:20 +0300 Subject: [PATCH 057/548] used logger custom interface instead of grpclog.LoggerV2 --- logger.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/logger.go b/logger.go index 21588e1..3bb969e 100644 --- a/logger.go +++ b/logger.go @@ -10,10 +10,17 @@ import ( "go.uber.org/zap/zapcore" ) -type zapLogger struct { - zapcore.Core - log *zap.SugaredLogger -} +type ( + zapLogger struct { + zapcore.Core + log *zap.SugaredLogger + } + + logger interface { + grpclog.LoggerV2 + Println(v ...interface{}) + } +) const ( formatJSON = "json" @@ -23,7 +30,7 @@ const ( defaultSamplingThereafter = 100 ) -func gRPCLogger(l *zap.Logger) grpclog.LoggerV2 { +func gRPCLogger(l *zap.Logger) logger { log := l.WithOptions( // skip gRPCLog + zapLogger in caller zap.AddCallerSkip(2)) From d0901587e266a01efe56bd47810cf6835f4a0d41 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 3 Mar 2020 13:36:52 +0300 Subject: [PATCH 058/548] simplify main func --- main.go | 41 ++++------------------------------------- 1 file changed, 4 insertions(+), 37 deletions(-) diff --git a/main.go b/main.go index ad17d6e..8a66199 100644 --- a/main.go +++ b/main.go @@ -7,8 +7,6 @@ import ( "time" "github.com/fasthttp/router" - "github.com/prometheus/client_golang/prometheus" - http "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/valyala/fasthttp" "go.uber.org/zap" "google.golang.org/grpc/grpclog" @@ -21,11 +19,6 @@ type app struct { key *ecdsa.PrivateKey } -const ( - defaultHealthyMsg = "NeoFS HTTP Gateway is " - defaultContentType = "text/plain; charset=utf-8" -) - func main() { var ( err error @@ -52,45 +45,19 @@ func main() { r.RedirectTrailingSlash = true r.GET("/get/:cid/:oid", a.receiveFile) - r.GET("/-/ready", func(ctx *fasthttp.RequestCtx) { - ctx.SetStatusCode(fasthttp.StatusOK) - ctx.SetBodyString("NeoFS HTTP Gateway is ready") - }) - - r.GET("/-/healthy", func(c *fasthttp.RequestCtx) { - code := fasthttp.StatusOK - msg := "healthy" - - if err := a.pool.unhealthy.Load(); err != nil { - msg = "unhealthy: " + err.Error() - code = fasthttp.StatusBadRequest - } - - c.Response.Reset() - c.SetStatusCode(code) - c.SetContentType(defaultContentType) - c.SetBodyString(defaultHealthyMsg + msg) - }) + // attaching /-/(ready,healthy) + attachHealthy(r, a.pool.unhealthy) // enable metrics if v.GetBool("metrics") { l.Info("enabled /metrics") - r.GET("/metrics/", metricsHandler(prometheus.DefaultGatherer, http.HandlerOpts{ - ErrorLog: z.(http.Logger), - //ErrorHandling: 0, - //Registry: nil, - //DisableCompression: false, - //MaxRequestsInFlight: 0, - //Timeout: 0, - //EnableOpenMetrics: false, - })) + attachMetrics(r, z) } // enable pprof if v.GetBool("pprof") { l.Info("enabled /debug/pprof") - r.GET("/debug/pprof/", pprofHandler()) - r.GET("/debug/pprof/:name", pprofHandler()) + attachProfiler(r) } en := &fasthttp.Server{ From 6abd79aa4409a5383b1bc29ef2d6ecab039351f6 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 10 Mar 2020 14:06:06 +0300 Subject: [PATCH 059/548] Update dependencies - github.com/nspcc-dev/neofs-api v0.4.1 // v0.4.0 => v0.4.1 - github.com/nspcc-dev/neofs-crypto v0.3.0 // v0.2.3 => v0.3.0 - github.com/prometheus/client_golang v1.5.0 // v1.4.1 => v1.5.0 --- go.mod | 6 +++--- go.sum | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index c7bdc19..cd245b7 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.13 require ( github.com/fasthttp/router v0.6.1 - github.com/nspcc-dev/neofs-api v0.4.0 - github.com/nspcc-dev/neofs-crypto v0.2.3 - github.com/prometheus/client_golang v1.4.1 + github.com/nspcc-dev/neofs-api v0.4.1 + github.com/nspcc-dev/neofs-crypto v0.3.0 + github.com/prometheus/client_golang v1.5.0 github.com/prometheus/common v0.9.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.6.2 diff --git a/go.sum b/go.sum index d518ce8..b18ac63 100644 --- a/go.sum +++ b/go.sum @@ -116,10 +116,10 @@ github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neofs-api v0.4.0 h1:SwoJ9XV9KxkIc7Nf5WOvMW0XIK6ZxRNXnGVNzp/uQdA= -github.com/nspcc-dev/neofs-api v0.4.0/go.mod h1:Y1PQ6nEZyA6sZPDHJxgPWWeT1oU2bKL1N0rpFQ4B/fk= -github.com/nspcc-dev/neofs-crypto v0.2.3 h1:aca3X2aly92ENRbFK+kH6Hd+J9EQ4Eu6XMVoITSIKtc= -github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= +github.com/nspcc-dev/neofs-api v0.4.1 h1:Kcr9mVmtCQPoPeHIL5z6dvE8P0zsispP2c2z6HA5bLM= +github.com/nspcc-dev/neofs-api v0.4.1/go.mod h1:kkoWJ6YnTRwRHI+h2kXbx21i/sZZs0LheF+T/33xzr4= +github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= +github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= github.com/nspcc-dev/netmap v1.6.1/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= @@ -140,6 +140,8 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= +github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= From 164b0870deefc0763779116736a1c566db4b9b2f Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 31 Mar 2020 11:37:10 +0300 Subject: [PATCH 060/548] refactoring + moved to neofs-api-go --- app.go | 207 ++++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- health.go | 4 +- main.go | 118 ++---------------------------- pool.go | 27 ++++++- pprof.go | 2 +- receive.go | 12 +-- settings.go | 2 +- 9 files changed, 252 insertions(+), 126 deletions(-) create mode 100644 app.go diff --git a/app.go b/app.go new file mode 100644 index 0000000..2b0b0e0 --- /dev/null +++ b/app.go @@ -0,0 +1,207 @@ +package main + +import ( + "context" + "crypto/ecdsa" + "errors" + "time" + + "github.com/fasthttp/router" + + "google.golang.org/grpc/grpclog" + + "github.com/nspcc-dev/neofs-api-go/service" + "github.com/nspcc-dev/neofs-api-go/state" + "github.com/spf13/viper" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +type ( + app struct { + pool *Pool + log *zap.Logger + cfg *viper.Viper + key *ecdsa.PrivateKey + + wlog logger + web *fasthttp.Server + + jobDone chan struct{} + webDone chan struct{} + + rebalanceTimer time.Duration + + nodes []string + + reqHealth *state.HealthRequest + reqNetmap *state.NetmapRequest + + conTimeout time.Duration + reqTimeout time.Duration + } + + App interface { + Wait() + Worker(context.Context) + Serve(context.Context) + } + + Option func(a *app) +) + +func WithLogger(l *zap.Logger) Option { + return func(a *app) { + if l == nil { + return + } + a.log = l + } +} + +func WithConfig(c *viper.Viper) Option { + return func(a *app) { + if c == nil { + return + } + a.cfg = c + } +} + +func newApp(opt ...Option) App { + a := &app{ + log: zap.L(), + cfg: viper.GetViper(), + web: new(fasthttp.Server), + + jobDone: make(chan struct{}), + webDone: make(chan struct{}), + } + + for i := range opt { + opt[i](a) + } + + a.wlog = gRPCLogger(a.log) + + if a.cfg.GetBool("verbose") { + grpclog.SetLoggerV2(a.wlog) + } + + a.key = fetchKey(a.log, a.cfg) + a.rebalanceTimer = a.cfg.GetDuration("rebalance_timer") + a.conTimeout = a.cfg.GetDuration("connect_timeout") + a.reqTimeout = a.cfg.GetDuration("request_timeout") + + // -- setup FastHTTP server: -- + a.web.Name = "neofs-http-gate" + a.web.ReadBufferSize = a.cfg.GetInt("web.read_buffer_size") + a.web.WriteBufferSize = a.cfg.GetInt("web.write_buffer_size") + a.web.ReadTimeout = a.cfg.GetDuration("web.read_timeout") + a.web.WriteTimeout = a.cfg.GetDuration("web.write_timeout") + a.web.GetOnly = true + a.web.DisableHeaderNamesNormalizing = true + a.web.NoDefaultServerHeader = true + a.web.NoDefaultContentType = true + // -- -- -- -- -- -- -- -- -- -- + + a.reqHealth = new(state.HealthRequest) + a.reqHealth.SetTTL(service.NonForwardingTTL) + + if err := service.SignRequestHeader(a.key, a.reqHealth); err != nil { + a.log.Fatal("could not sign `HealthRequest`", zap.Error(err)) + } + + a.reqNetmap = new(state.NetmapRequest) + a.reqNetmap.SetTTL(service.SingleForwardingTTL) + + if err := service.SignRequestHeader(a.key, a.reqNetmap); err != nil { + a.log.Fatal("could not sign `NetmapRequest`", zap.Error(err)) + } + + a.pool = newPool(a.log, a.cfg) + + return a +} + +func (a *app) Wait() { + a.log.Info("application started") + + select { + case <-a.jobDone: // wait for job is stopped + <-a.webDone + case <-a.webDone: // wait for web-server is stopped + <-a.jobDone + } +} + +func (a *app) Worker(ctx context.Context) { + dur := a.rebalanceTimer + tick := time.NewTimer(dur) + + a.pool.reBalance(ctx) + + switch _, err := a.pool.getConnection(ctx); { + case err == nil: + // ignore + case errors.Is(err, context.Canceled): + // ignore + // l.Info("context canceled") + default: + a.log.Fatal("could get connection", zap.Error(err)) + } + +loop: + for { + select { + case <-ctx.Done(): + break loop + case <-tick.C: + a.pool.reBalance(ctx) + tick.Reset(dur) + } + } + + a.pool.close() + tick.Stop() + + a.log.Info("connection worker stopped") + + close(a.jobDone) +} + +func (a *app) Serve(ctx context.Context) { + go func() { + <-ctx.Done() + a.log.Info("stop web-server", zap.Error(a.web.Shutdown())) + close(a.webDone) + }() + + r := router.New() + r.RedirectTrailingSlash = true + r.GET("/get/:cid/:oid/", a.receiveFile) + + // attaching /-/(ready,healthy) + attachHealthy(r, a.pool.unhealthy) + + // enable metrics + if a.cfg.GetBool("metrics") { + a.log.Info("enabled /metrics") + attachMetrics(r, a.wlog) + } + + // enable pprof + if a.cfg.GetBool("pprof") { + a.log.Info("enabled /debug/pprof") + attachProfiler(r) + } + + bind := a.cfg.GetString("listen_address") + a.log.Info("run gateway server", + zap.String("address", bind)) + + a.web.Handler = r.Handler + if err := a.web.ListenAndServe(bind); err != nil { + a.log.Fatal("could not start server", zap.Error(err)) + } +} diff --git a/go.mod b/go.mod index cd245b7..3a214b6 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/fasthttp/router v0.6.1 - github.com/nspcc-dev/neofs-api v0.4.1 + github.com/nspcc-dev/neofs-api-go v0.5.0 github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/prometheus/client_golang v1.5.0 github.com/prometheus/common v0.9.1 diff --git a/go.sum b/go.sum index b18ac63..bd7466b 100644 --- a/go.sum +++ b/go.sum @@ -116,8 +116,8 @@ github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neofs-api v0.4.1 h1:Kcr9mVmtCQPoPeHIL5z6dvE8P0zsispP2c2z6HA5bLM= -github.com/nspcc-dev/neofs-api v0.4.1/go.mod h1:kkoWJ6YnTRwRHI+h2kXbx21i/sZZs0LheF+T/33xzr4= +github.com/nspcc-dev/neofs-api-go v0.5.0 h1:YGX2lEjFmiFYmk8gCkLINJjSh6hToKp2oo0sG5u8G+4= +github.com/nspcc-dev/neofs-api-go v0.5.0/go.mod h1:RlP++uLPHmXN81KWmdgBujo6FEeTn/b3aoQjHgKuNd4= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= diff --git a/health.go b/health.go index 708bbb7..3141a0f 100644 --- a/health.go +++ b/health.go @@ -12,12 +12,12 @@ const ( ) func attachHealthy(r *router.Router, e *atomic.Error) { - r.GET("/-/ready", func(ctx *fasthttp.RequestCtx) { + r.GET("/-/ready/", func(ctx *fasthttp.RequestCtx) { ctx.SetStatusCode(fasthttp.StatusOK) ctx.SetBodyString(healthyState + "ready") }) - r.GET("/-/healthy", func(c *fasthttp.RequestCtx) { + r.GET("/-/healthy/", func(c *fasthttp.RequestCtx) { code := fasthttp.StatusOK msg := "healthy" diff --git a/main.go b/main.go index 8a66199..1d9c832 100644 --- a/main.go +++ b/main.go @@ -1,122 +1,18 @@ package main -import ( - "context" - "crypto/ecdsa" - "errors" - "time" - - "github.com/fasthttp/router" - "github.com/valyala/fasthttp" - "go.uber.org/zap" - "google.golang.org/grpc/grpclog" -) - -type app struct { - pool *Pool - log *zap.Logger - timeout time.Duration - key *ecdsa.PrivateKey -} - func main() { var ( - err error - v = settings() l = newLogger(v) - z = gRPCLogger(l) g = newGracefulContext(l) - d = v.GetDuration("rebalance_timer") + + a = newApp( + WithLogger(l), + WithConfig(v)) ) - if v.GetBool("verbose") { - grpclog.SetLoggerV2(z) - } + go a.Serve(g) + go a.Worker(g) - a := &app{ - log: l, - pool: newPool(l, v), - key: fetchKey(l, v), - timeout: v.GetDuration("request_timeout"), - } - - r := router.New() - r.RedirectTrailingSlash = true - r.GET("/get/:cid/:oid", a.receiveFile) - - // attaching /-/(ready,healthy) - attachHealthy(r, a.pool.unhealthy) - - // enable metrics - if v.GetBool("metrics") { - l.Info("enabled /metrics") - attachMetrics(r, z) - } - - // enable pprof - if v.GetBool("pprof") { - l.Info("enabled /debug/pprof") - attachProfiler(r) - } - - en := &fasthttp.Server{ - Name: "neofs-http-gate", - Handler: r.Handler, - ReadBufferSize: 4096, - ReadTimeout: time.Second * 15, - GetOnly: true, - DisableHeaderNamesNormalizing: true, - NoDefaultServerHeader: true, - NoDefaultContentType: true, - } - - go func() { - bind := v.GetString("listen_address") - l.Info("run gateway server", - zap.String("address", bind)) - - if err := en.ListenAndServe(bind); err != nil { - l.Panic("could not start server", zap.Error(err)) - } - }() - - go checkConnection(g, d, a.pool) - - a.pool.reBalance(g) - - switch _, err = a.pool.getConnection(g); { - case err == nil: - // ignore - case errors.Is(err, context.Canceled): - // ignore - // l.Info("context canceled") - default: - l.Error("could get connection", zap.Error(err)) - return - } - - <-g.Done() - - l.Info("web server stopped", zap.Error(en.Shutdown())) -} - -func checkConnection(ctx context.Context, dur time.Duration, p *Pool) { - tick := time.NewTimer(dur) - -loop: - for { - select { - case <-ctx.Done(): - break loop - case <-tick.C: - p.reBalance(ctx) - tick.Reset(dur) - } - } - - p.close() - tick.Stop() - - p.log.Info("connection worker stopped") + a.Wait() } diff --git a/pool.go b/pool.go index 7baa273..15f9f47 100644 --- a/pool.go +++ b/pool.go @@ -11,8 +11,8 @@ import ( "sync" "time" - "github.com/nspcc-dev/neofs-api/service" - "github.com/nspcc-dev/neofs-api/state" + "github.com/nspcc-dev/neofs-api-go/service" + "github.com/nspcc-dev/neofs-api-go/state" "github.com/spf13/viper" "go.uber.org/atomic" "go.uber.org/zap" @@ -149,6 +149,29 @@ func (p *Pool) close() { } } +func (a *app) checkHealth(ctx context.Context, conn *grpc.ClientConn) error { + //a.log.Info("try to fetch node health status", + // zap.String("node", conn.Target()), + // zap.Stringer("timeout", a.reqTimeout)) + + ctx, cancel := context.WithTimeout(ctx, a.reqTimeout) + result, err := state.NewStatusClient(conn).HealthCheck(ctx, a.reqHealth) + cancel() + + if err != nil { + result = &state.HealthResponse{Status: err.Error()} + } else if !result.Healthy { + err = errors.New(result.Status) + } + + a.log.Debug("received node health status", + zap.String("node", conn.Target()), + zap.String("status", result.Status), + zap.Error(err)) + + return err +} + func (p *Pool) reBalance(ctx context.Context) { p.Lock() defer func() { diff --git a/pprof.go b/pprof.go index ef5457a..7400ea5 100644 --- a/pprof.go +++ b/pprof.go @@ -11,7 +11,7 @@ import ( func attachProfiler(r *router.Router) { r.GET("/debug/pprof/", pprofHandler()) - r.GET("/debug/pprof/:name", pprofHandler()) + r.GET("/debug/pprof/:name/", pprofHandler()) } func pprofHandler() fasthttp.RequestHandler { diff --git a/receive.go b/receive.go index a3d001b..de2d148 100644 --- a/receive.go +++ b/receive.go @@ -9,10 +9,10 @@ import ( "strings" "time" - "github.com/nspcc-dev/neofs-api/container" - "github.com/nspcc-dev/neofs-api/object" - "github.com/nspcc-dev/neofs-api/refs" - "github.com/nspcc-dev/neofs-api/service" + "github.com/nspcc-dev/neofs-api-go/container" + "github.com/nspcc-dev/neofs-api-go/object" + "github.com/nspcc-dev/neofs-api-go/refs" + "github.com/nspcc-dev/neofs-api-go/service" "github.com/valyala/fasthttp" "go.uber.org/zap" "google.golang.org/grpc" @@ -49,7 +49,7 @@ func (a *app) receiveFile(c *fasthttp.RequestCtx) { } { // try to connect or throw http error: - ctx, cancel := context.WithTimeout(c, a.timeout) + ctx, cancel := context.WithTimeout(c, a.reqTimeout) defer cancel() if con, err = a.pool.getConnection(ctx); err != nil { @@ -59,7 +59,7 @@ func (a *app) receiveFile(c *fasthttp.RequestCtx) { } } - ctx, cancel := context.WithTimeout(c, a.timeout) + ctx, cancel := context.WithTimeout(c, a.reqTimeout) defer cancel() log = log.With(zap.String("node", con.Target())) diff --git a/settings.go b/settings.go index f21bfd9..02cba71 100644 --- a/settings.go +++ b/settings.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/nspcc-dev/neofs-api/refs" + "github.com/nspcc-dev/neofs-api-go/refs" crypto "github.com/nspcc-dev/neofs-crypto" "github.com/spf13/pflag" "github.com/spf13/viper" From ea4710eabac7332a8dd08f34de50ed357b788871 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 15 Apr 2020 11:10:42 +0300 Subject: [PATCH 061/548] update to new router --- app.go | 2 +- go.mod | 6 +++--- go.sum | 9 +++++++++ settings.go | 7 +++++++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/app.go b/app.go index 2b0b0e0..0434e0a 100644 --- a/app.go +++ b/app.go @@ -179,7 +179,7 @@ func (a *app) Serve(ctx context.Context) { r := router.New() r.RedirectTrailingSlash = true - r.GET("/get/:cid/:oid/", a.receiveFile) + r.GET("/get/{cid}/{oid}/", a.receiveFile) // attaching /-/(ready,healthy) attachHealthy(r, a.pool.unhealthy) diff --git a/go.mod b/go.mod index 3a214b6..b92b71e 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/nspcc-dev/neofs-gw go 1.13 require ( - github.com/fasthttp/router v0.6.1 + github.com/fasthttp/router v1.0.2 github.com/nspcc-dev/neofs-api-go v0.5.0 github.com/nspcc-dev/neofs-crypto v0.3.0 - github.com/prometheus/client_golang v1.5.0 + github.com/prometheus/client_golang v1.5.1 github.com/prometheus/common v0.9.1 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.6.2 + github.com/spf13/viper v1.6.3 github.com/valyala/fasthttp v1.9.0 go.uber.org/atomic v1.6.0 go.uber.org/zap v1.14.0 diff --git a/go.sum b/go.sum index bd7466b..ee0b963 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -37,6 +38,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fasthttp/router v0.6.1 h1:cPfY4S9tZSh0J62O6h4n6Kxwg9eskQ2GPCNWvXDsa1s= github.com/fasthttp/router v0.6.1/go.mod h1:00BQmm3xiThNypescxIQ+Gfgw2I/3QWKvuagFoENUb4= +github.com/fasthttp/router v1.0.2 h1:rdYdcAmwOLqWuFgc4afa409SYmuw4t0A66K5Ib+GT3I= +github.com/fasthttp/router v1.0.2/go.mod h1:Myk/ofrwtfiLSCIfbE44+e+PyP3mR6JhZg3AYzqwJI0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -142,6 +145,8 @@ github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFY github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= +github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -162,6 +167,8 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f h1:PgA+Olipyj258EIEYnpFFONrrCcAIWNUNoFhUfMqAGY= github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= +github.com/savsgio/gotils v0.0.0-20200319105752-a9cc718f6a3f h1:XfUnevLK4O22at3R77FlyQHKwlQs75LELdsH2wRX2KQ= +github.com/savsgio/gotils v0.0.0-20200319105752-a9cc718f6a3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -183,6 +190,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= +github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/settings.go b/settings.go index 02cba71..d20e54c 100644 --- a/settings.go +++ b/settings.go @@ -114,6 +114,13 @@ func settings() *viper.Viper { v.SetDefault("keepalive.timeout", defaultKeepaliveTimeout) v.SetDefault("keepalive.permit_without_stream", true) + // web-server: + v.SetDefault("web.read_buffer_size", 4096) + v.SetDefault("web.write_buffer_size", 4096) + v.SetDefault("web.read_timeout", time.Second*15) + v.SetDefault("web.write_timeout", time.Minute) + v.SetDefault("web.connection_per_host", 10) + if err := v.BindPFlags(flags); err != nil { panic(err) } From dc3d041ad685c819e0b98d21840d78043781b04e Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 15 Apr 2020 11:17:58 +0300 Subject: [PATCH 062/548] fixes for Dockerfile --- Dockerfile | 20 +++++++++++--------- go.sum | 6 ------ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3777b08..a6f62ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,24 @@ -FROM golang:1-alpine as builder +FROM golang:1 as builder -ARG BUILD=now -ARG VERSION=dev -ARG REPO=github.com/nspcc-dev/neofs-gw ENV GOGC off ENV CGO_ENABLED 0 -# add later -w -s -ENV LDFLAGS " -X main.Version=${VERSION} -compressdwarf=false" + +RUN set -x \ + && apt update \ + && apt install -y upx-ucl WORKDIR /src - COPY . /src -RUN go build -v -mod=vendor -trimpath -gcflags=all="-N -l" -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N)" -o /go/bin/neofs-gw ./ +ARG VERSION=dev +ENV LDFLAGS "-w -s -X main.Version=${VERSION}" +RUN set -x \ + && go build -v -mod=vendor -trimpath -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N)" -o /go/bin/neofs-gw ./ \ + && upx -3 /go/bin/neofs-gw # Executable image -FROM alpine +FROM scratch WORKDIR / diff --git a/go.sum b/go.sum index ee0b963..ac8d11b 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,6 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fasthttp/router v0.6.1 h1:cPfY4S9tZSh0J62O6h4n6Kxwg9eskQ2GPCNWvXDsa1s= -github.com/fasthttp/router v0.6.1/go.mod h1:00BQmm3xiThNypescxIQ+Gfgw2I/3QWKvuagFoENUb4= github.com/fasthttp/router v1.0.2 h1:rdYdcAmwOLqWuFgc4afa409SYmuw4t0A66K5Ib+GT3I= github.com/fasthttp/router v1.0.2/go.mod h1:Myk/ofrwtfiLSCIfbE44+e+PyP3mR6JhZg3AYzqwJI0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -143,8 +141,6 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= -github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -165,8 +161,6 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f h1:PgA+Olipyj258EIEYnpFFONrrCcAIWNUNoFhUfMqAGY= -github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= github.com/savsgio/gotils v0.0.0-20200319105752-a9cc718f6a3f h1:XfUnevLK4O22at3R77FlyQHKwlQs75LELdsH2wRX2KQ= github.com/savsgio/gotils v0.0.0-20200319105752-a9cc718f6a3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= From 96e24653f7e510eda52d59f2fac0ff6ed2b634b3 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 22 Apr 2020 13:16:58 +0300 Subject: [PATCH 063/548] modules: update to neofs-api 0.7.1 --- go.mod | 4 ++-- go.sum | 34 +++++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index b92b71e..80ed0e6 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/fasthttp/router v1.0.2 - github.com/nspcc-dev/neofs-api-go v0.5.0 + github.com/nspcc-dev/neofs-api-go v0.7.1 github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/prometheus/client_golang v1.5.1 github.com/prometheus/common v0.9.1 @@ -16,5 +16,5 @@ require ( golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 // indirect golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect golang.org/x/text v0.3.2 // indirect - google.golang.org/grpc v1.27.1 + google.golang.org/grpc v1.28.1 ) diff --git a/go.sum b/go.sum index ac8d11b..d4ce4fb 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,7 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -34,7 +35,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fasthttp/router v1.0.2 h1:rdYdcAmwOLqWuFgc4afa409SYmuw4t0A66K5Ib+GT3I= github.com/fasthttp/router v1.0.2/go.mod h1:Myk/ofrwtfiLSCIfbE44+e+PyP3mR6JhZg3AYzqwJI0= @@ -62,8 +64,15 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -117,12 +126,12 @@ github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neofs-api-go v0.5.0 h1:YGX2lEjFmiFYmk8gCkLINJjSh6hToKp2oo0sG5u8G+4= -github.com/nspcc-dev/neofs-api-go v0.5.0/go.mod h1:RlP++uLPHmXN81KWmdgBujo6FEeTn/b3aoQjHgKuNd4= +github.com/nspcc-dev/neofs-api-go v0.7.1 h1:Q26tsfJN9vLyAaJHPYr/ism8VOWRdMYPnIO9QeNrwKQ= +github.com/nspcc-dev/neofs-api-go v0.7.1/go.mod h1:fzV1gSigr5sEYWjs1jL395Cyv+6y/dMk2PeVPv+Vqlo= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/netmap v1.6.1 h1:Pigqpqi6QSdRiusbq5XlO20A18k6Eyu7j9MzOfAE3CM= -github.com/nspcc-dev/netmap v1.6.1/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= +github.com/nspcc-dev/netmap v1.7.0 h1:ak64xn/gPdgYw4tsqSSF7kAGQGbEpeuJEF3XwBX4L9Y= +github.com/nspcc-dev/netmap v1.7.0/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/tzhash v1.4.0 h1:RVIR+mxOBHl58CE99+DXtE31ylD5PEkZSoWqoj4fVjg= @@ -139,8 +148,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= -github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -192,6 +199,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -287,8 +296,15 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k= +google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 12a3801f3dd5ad17f359a2d02f6fc03312a3c3d9 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 22 Apr 2020 13:34:48 +0300 Subject: [PATCH 064/548] errors: correct handling 404 --- go.mod | 1 + receive.go | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 80ed0e6..ed473cb 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/fasthttp/router v1.0.2 github.com/nspcc-dev/neofs-api-go v0.7.1 github.com/nspcc-dev/neofs-crypto v0.3.0 + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.5.1 github.com/prometheus/common v0.9.1 github.com/spf13/pflag v1.0.5 diff --git a/receive.go b/receive.go index de2d148..7f09ff4 100644 --- a/receive.go +++ b/receive.go @@ -6,16 +6,17 @@ import ( "net/http" "path" "strconv" - "strings" "time" - "github.com/nspcc-dev/neofs-api-go/container" "github.com/nspcc-dev/neofs-api-go/object" "github.com/nspcc-dev/neofs-api-go/refs" "github.com/nspcc-dev/neofs-api-go/service" + "github.com/pkg/errors" "github.com/valyala/fasthttp" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func (a *app) receiveFile(c *fasthttp.RequestCtx) { @@ -91,14 +92,20 @@ func (a *app) receiveFile(c *fasthttp.RequestCtx) { zap.Stringer("elapsed", time.Since(start)), zap.Error(err)) - switch { - case strings.Contains(err.Error(), object.ErrNotFound.Error()), - strings.Contains(err.Error(), container.ErrNotFound.Error()): - c.Error("object not found", fasthttp.StatusNotFound) - default: - c.Error("could not receive object", fasthttp.StatusBadRequest) + var ( + msg = errors.Wrap(err, "could not receive object").Error() + code = fasthttp.StatusBadRequest + ) + + if st, ok := status.FromError(errors.Cause(err)); ok && st != nil { + if st.Code() == codes.NotFound { + code = fasthttp.StatusNotFound + } + + msg = st.Message() } + c.Error(msg, code) return } } From 54071b5da46175ce7aeed04f9929e3269955dc4b Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 12 May 2020 11:18:23 +0300 Subject: [PATCH 065/548] routing: remove trailing slash --- app.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app.go b/app.go index 0434e0a..23d1860 100644 --- a/app.go +++ b/app.go @@ -179,7 +179,9 @@ func (a *app) Serve(ctx context.Context) { r := router.New() r.RedirectTrailingSlash = true - r.GET("/get/{cid}/{oid}/", a.receiveFile) + + a.log.Info("enabled /get/{cid}/{oid}") + r.GET("/get/{cid}/{oid}", a.receiveFile) // attaching /-/(ready,healthy) attachHealthy(r, a.pool.unhealthy) From 21fc1c6e6c78c719cac0b469776d4f489fa9930c Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 12 May 2020 11:19:07 +0300 Subject: [PATCH 066/548] log: fix routes in log message --- app.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.go b/app.go index 23d1860..e5ae490 100644 --- a/app.go +++ b/app.go @@ -188,13 +188,13 @@ func (a *app) Serve(ctx context.Context) { // enable metrics if a.cfg.GetBool("metrics") { - a.log.Info("enabled /metrics") + a.log.Info("enabled /metrics/") attachMetrics(r, a.wlog) } // enable pprof if a.cfg.GetBool("pprof") { - a.log.Info("enabled /debug/pprof") + a.log.Info("enabled /debug/pprof/") attachProfiler(r) } From 0a55a84863bc5ba3d93d56dabc49fc5120cf25c9 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 12 May 2020 11:19:34 +0300 Subject: [PATCH 067/548] metrics: fix doc-comments --- metrics.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/metrics.go b/metrics.go index 4a7c857..e888594 100644 --- a/metrics.go +++ b/metrics.go @@ -13,12 +13,12 @@ import ( func attachMetrics(r *router.Router, z promhttp.Logger) { r.GET("/metrics/", metricsHandler(prometheus.DefaultGatherer, promhttp.HandlerOpts{ ErrorLog: z, - //ErrorHandling: 0, - //Registry: nil, - //DisableCompression: false, - //MaxRequestsInFlight: 0, - //Timeout: 0, - //EnableOpenMetrics: false, + // ErrorHandling: 0, + // Registry: nil, + // DisableCompression: false, + // MaxRequestsInFlight: 0, + // Timeout: 0, + // EnableOpenMetrics: false, })) } From 39e4a770bbd5b331098348afa523f13be99cb574 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 12 May 2020 11:20:00 +0300 Subject: [PATCH 068/548] pprof: fix route params --- pprof.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pprof.go b/pprof.go index 7400ea5..0ef1fe5 100644 --- a/pprof.go +++ b/pprof.go @@ -11,7 +11,7 @@ import ( func attachProfiler(r *router.Router) { r.GET("/debug/pprof/", pprofHandler()) - r.GET("/debug/pprof/:name/", pprofHandler()) + r.GET("/debug/pprof/{name}/", pprofHandler()) } func pprofHandler() fasthttp.RequestHandler { From a9f4ee535a0c5421142969ec2ca575f324ef3acd Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 3 Jun 2020 06:30:36 +0300 Subject: [PATCH 069/548] Update dependencies - github.com/fasthttp/router v1.1.6 - github.com/nspcc-dev/neofs-api-go v1.0.0 - github.com/prometheus/client_golang v1.6.0 - github.com/prometheus/common v0.10.0 - github.com/spf13/viper v1.7.0 - github.com/valyala/fasthttp v1.14.0 - go.uber.org/zap v1.15.0 - google.golang.org/grpc v1.29.1 --- go.mod | 18 +++-- go.sum | 203 +++++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 170 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index ed473cb..8ed1c0a 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,17 @@ module github.com/nspcc-dev/neofs-gw go 1.13 require ( - github.com/fasthttp/router v1.0.2 - github.com/nspcc-dev/neofs-api-go v0.7.1 + github.com/fasthttp/router v1.1.6 + github.com/nspcc-dev/neofs-api-go v1.0.0 github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.5.1 - github.com/prometheus/common v0.9.1 + github.com/prometheus/client_golang v1.6.0 + github.com/prometheus/common v0.10.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.6.3 - github.com/valyala/fasthttp v1.9.0 + github.com/spf13/viper v1.7.0 + github.com/valyala/fasthttp v1.14.0 go.uber.org/atomic v1.6.0 - go.uber.org/zap v1.14.0 + go.uber.org/zap v1.15.0 golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 // indirect - golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect - golang.org/x/text v0.3.2 // indirect - google.golang.org/grpc v1.28.1 + google.golang.org/grpc v1.29.1 ) diff --git a/go.sum b/go.sum index d4ce4fb..70d1486 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,19 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= @@ -8,13 +21,19 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= +github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310 h1:t+qxRrRtwNiUYA+Xh2jSXhoG2grnMCMKX4Fg6lx9X1U= github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -25,9 +44,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -38,13 +56,14 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fasthttp/router v1.0.2 h1:rdYdcAmwOLqWuFgc4afa409SYmuw4t0A66K5Ib+GT3I= -github.com/fasthttp/router v1.0.2/go.mod h1:Myk/ofrwtfiLSCIfbE44+e+PyP3mR6JhZg3AYzqwJI0= +github.com/fasthttp/router v1.1.6 h1:lBcXxp1ZNoNbSeh4+RvAaXKSEiHU6sGd+gEMpd5Xjog= +github.com/fasthttp/router v1.1.6/go.mod h1:E1mpv7mrQzAhiSQdqhRb+GBTC7MEV+bLFVmgzSA5oFM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -59,6 +78,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -70,6 +91,9 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -77,30 +101,53 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.8.2 h1:Bx0qjetmNjdFXASH02NSAREKpiaDwkO1DRZ3dV2KCcs= -github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/compress v1.10.4 h1:jFzIFaf586tquEB5EhzQG0HwGNSlgAJpG53G6Ss11wc= +github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -111,9 +158,17 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -126,8 +181,8 @@ github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neofs-api-go v0.7.1 h1:Q26tsfJN9vLyAaJHPYr/ism8VOWRdMYPnIO9QeNrwKQ= -github.com/nspcc-dev/neofs-api-go v0.7.1/go.mod h1:fzV1gSigr5sEYWjs1jL395Cyv+6y/dMk2PeVPv+Vqlo= +github.com/nspcc-dev/neofs-api-go v1.0.0 h1:FSsV3UFjTfBjaqgaOW7pZ9J8wqg4uOEF+DGfH04sY/M= +github.com/nspcc-dev/neofs-api-go v1.0.0/go.mod h1:2tf31g2Ns/Z2ev5d8LZ/9f1VHIeY5LHpDbq4EsDhYM0= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/netmap v1.7.0 h1:ak64xn/gPdgYw4tsqSSF7kAGQGbEpeuJEF3XwBX4L9Y= @@ -137,6 +192,7 @@ github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452 github.com/nspcc-dev/tzhash v1.4.0 h1:RVIR+mxOBHl58CE99+DXtE31ylD5PEkZSoWqoj4fVjg= github.com/nspcc-dev/tzhash v1.4.0/go.mod h1:Z8gp/VZbyWgPhaMp/KTmeoW5UTynp/N60g0jTtSzBws= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -145,11 +201,12 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= +github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -160,16 +217,20 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/savsgio/gotils v0.0.0-20200319105752-a9cc718f6a3f h1:XfUnevLK4O22at3R77FlyQHKwlQs75LELdsH2wRX2KQ= -github.com/savsgio/gotils v0.0.0-20200319105752-a9cc718f6a3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/savsgio/gotils v0.0.0-20200413113635-8c468ce75cca h1:Qe7Mtuhjkk38HVpRtvWdziZJcwG3Qup1mfyvyOrcnyM= +github.com/savsgio/gotils v0.0.0-20200413113635-8c468ce75cca/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -189,10 +250,8 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= -github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= -github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= -github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -204,74 +263,104 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw= -github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasthttp v1.14.0 h1:67bfuW9azCMwW/Jlq/C+VeihNpAuJMWkYPBig1gdi3A= +github.com/valyala/fasthttp v1.14.0/go.mod h1:ol1PCaL0dX20wC0htZ7sYCsvCYmrouYra0zHzaclZhE= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.14.0 h1:/pduUoebOeeJzTDFuoMgC6nRkiasr1sBCIEorly7m4o= -go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= +go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -279,32 +368,61 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k= -google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -322,6 +440,9 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= From 799d3915086b07d489b6616256921b8bb5330635 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 3 Jun 2020 06:30:58 +0300 Subject: [PATCH 070/548] migrate to new api v1.0.0 --- app.go | 4 ++-- receive.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app.go b/app.go index e5ae490..dc56ddc 100644 --- a/app.go +++ b/app.go @@ -108,14 +108,14 @@ func newApp(opt ...Option) App { a.reqHealth = new(state.HealthRequest) a.reqHealth.SetTTL(service.NonForwardingTTL) - if err := service.SignRequestHeader(a.key, a.reqHealth); err != nil { + if err := service.SignDataWithSessionToken(a.key, a.reqHealth); err != nil { a.log.Fatal("could not sign `HealthRequest`", zap.Error(err)) } a.reqNetmap = new(state.NetmapRequest) a.reqNetmap.SetTTL(service.SingleForwardingTTL) - if err := service.SignRequestHeader(a.key, a.reqNetmap); err != nil { + if err := service.SignDataWithSessionToken(a.key, a.reqNetmap); err != nil { a.log.Fatal("could not sign `NetmapRequest`", zap.Error(err)) } diff --git a/receive.go b/receive.go index 7f09ff4..2771bd7 100644 --- a/receive.go +++ b/receive.go @@ -76,7 +76,7 @@ func (a *app) receiveFile(c *fasthttp.RequestCtx) { req := &object.GetRequest{Address: refs.Address{ObjectID: oid, CID: cid}} req.SetTTL(service.SingleForwardingTTL) - if err = service.SignRequestHeader(a.key, req); err != nil { + if err = service.SignDataWithSessionToken(a.key, req); err != nil { log.Error("could not sign request", zap.Error(err)) c.Error("could not sign request", fasthttp.StatusBadRequest) return From 793d21651b7deb8770b8ea38ca9109cc1fbde148 Mon Sep 17 00:00:00 2001 From: "anatoly@nspcc.ru" Date: Wed, 10 Jun 2020 20:04:27 +0300 Subject: [PATCH 071/548] Add filename to save file in the browser --- receive.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/receive.go b/receive.go index 2771bd7..8a714b1 100644 --- a/receive.go +++ b/receive.go @@ -113,6 +113,7 @@ func (a *app) receiveFile(c *fasthttp.RequestCtx) { func receiveObject(c *fasthttp.RequestCtx, cli object.Service_GetClient) error { var ( typ string + content_disp_type string put = c.Request.URI().QueryArgs().GetBool("download") ) @@ -138,9 +139,12 @@ func receiveObject(c *fasthttp.RequestCtx, cli object.Service_GetClient) error { if hdr := obj.Headers[i].GetUserHeader(); hdr != nil { c.Response.Header.Set("x-"+hdr.Key, hdr.Value) - if hdr.Key == object.FilenameHeader && put { + if hdr.Key == object.FilenameHeader { + if put { + content_disp_type = "attachment" + } else { content_disp_type = "inline"} // NOTE: we use path.Base because hdr.Value can be something like `/path/to/filename.ext` - c.Response.Header.Set("Content-Disposition", "attachment; filename="+path.Base(hdr.Value)) + c.Response.Header.Set("Content-Disposition", content_disp_type+"; filename="+path.Base(hdr.Value)) } } } From f9e1e50da1ce0e2d6d313d59c400cd7a91e470f5 Mon Sep 17 00:00:00 2001 From: "anatoly@nspcc.ru" Date: Wed, 10 Jun 2020 20:28:09 +0300 Subject: [PATCH 072/548] update format --- receive.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/receive.go b/receive.go index 8a714b1..ad4ff19 100644 --- a/receive.go +++ b/receive.go @@ -142,7 +142,10 @@ func receiveObject(c *fasthttp.RequestCtx, cli object.Service_GetClient) error { if hdr.Key == object.FilenameHeader { if put { content_disp_type = "attachment" - } else { content_disp_type = "inline"} + } + else { + content_disp_type = "inline" + } // NOTE: we use path.Base because hdr.Value can be something like `/path/to/filename.ext` c.Response.Header.Set("Content-Disposition", content_disp_type+"; filename="+path.Base(hdr.Value)) } From 0e1fea52c6f5ed6963b1db617f9d7e245cb587ce Mon Sep 17 00:00:00 2001 From: "anatoly@nspcc.ru" Date: Wed, 10 Jun 2020 20:30:43 +0300 Subject: [PATCH 073/548] update format --- receive.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/receive.go b/receive.go index ad4ff19..f257290 100644 --- a/receive.go +++ b/receive.go @@ -142,8 +142,7 @@ func receiveObject(c *fasthttp.RequestCtx, cli object.Service_GetClient) error { if hdr.Key == object.FilenameHeader { if put { content_disp_type = "attachment" - } - else { + } else { content_disp_type = "inline" } // NOTE: we use path.Base because hdr.Value can be something like `/path/to/filename.ext` From 6b3344bd41c569a64d3a6fab6296833cf0b3546e Mon Sep 17 00:00:00 2001 From: "anatoly@nspcc.ru" Date: Wed, 10 Jun 2020 20:38:03 +0300 Subject: [PATCH 074/548] update --- receive.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/receive.go b/receive.go index f257290..58109e6 100644 --- a/receive.go +++ b/receive.go @@ -113,7 +113,7 @@ func (a *app) receiveFile(c *fasthttp.RequestCtx) { func receiveObject(c *fasthttp.RequestCtx, cli object.Service_GetClient) error { var ( typ string - content_disp_type string + content_disp_type string = "inline" put = c.Request.URI().QueryArgs().GetBool("download") ) @@ -142,9 +142,7 @@ func receiveObject(c *fasthttp.RequestCtx, cli object.Service_GetClient) error { if hdr.Key == object.FilenameHeader { if put { content_disp_type = "attachment" - } else { - content_disp_type = "inline" - } + } // NOTE: we use path.Base because hdr.Value can be something like `/path/to/filename.ext` c.Response.Header.Set("Content-Disposition", content_disp_type+"; filename="+path.Base(hdr.Value)) } From bbb2a057c7024448e7dcafbf7f372e6d97a5784a Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 2 Jul 2020 11:34:54 +0300 Subject: [PATCH 075/548] NFSSVC-16 Sign health-check requests --- app.go | 25 ++----------------------- pool.go | 56 ++++++++++++++++++++------------------------------------ 2 files changed, 22 insertions(+), 59 deletions(-) diff --git a/app.go b/app.go index dc56ddc..b255c9e 100644 --- a/app.go +++ b/app.go @@ -7,14 +7,10 @@ import ( "time" "github.com/fasthttp/router" - - "google.golang.org/grpc/grpclog" - - "github.com/nspcc-dev/neofs-api-go/service" - "github.com/nspcc-dev/neofs-api-go/state" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" + "google.golang.org/grpc/grpclog" ) type ( @@ -34,9 +30,6 @@ type ( nodes []string - reqHealth *state.HealthRequest - reqNetmap *state.NetmapRequest - conTimeout time.Duration reqTimeout time.Duration } @@ -105,21 +98,7 @@ func newApp(opt ...Option) App { a.web.NoDefaultContentType = true // -- -- -- -- -- -- -- -- -- -- - a.reqHealth = new(state.HealthRequest) - a.reqHealth.SetTTL(service.NonForwardingTTL) - - if err := service.SignDataWithSessionToken(a.key, a.reqHealth); err != nil { - a.log.Fatal("could not sign `HealthRequest`", zap.Error(err)) - } - - a.reqNetmap = new(state.NetmapRequest) - a.reqNetmap.SetTTL(service.SingleForwardingTTL) - - if err := service.SignDataWithSessionToken(a.key, a.reqNetmap); err != nil { - a.log.Fatal("could not sign `NetmapRequest`", zap.Error(err)) - } - - a.pool = newPool(a.log, a.cfg) + a.pool = newPool(a.log, a.cfg, a.key) return a } diff --git a/pool.go b/pool.go index 15f9f47..d45cd44 100644 --- a/pool.go +++ b/pool.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/ecdsa" crand "crypto/rand" "encoding/binary" "errors" @@ -35,9 +36,11 @@ type ( ttl time.Duration - connectTimeout time.Duration - requestTimeout time.Duration - opts keepalive.ClientParameters + conTimeout time.Duration + reqTimeout time.Duration + opts keepalive.ClientParameters + + reqHealth *state.HealthRequest currentIdx *atomic.Int32 currentConn *grpc.ClientConn @@ -57,7 +60,7 @@ var ( errNoHealthyConnections = errors.New("no active connections") ) -func newPool(l *zap.Logger, v *viper.Viper) *Pool { +func newPool(l *zap.Logger, v *viper.Viper, key *ecdsa.PrivateKey) *Pool { p := &Pool{ log: l, Mutex: new(sync.Mutex), @@ -70,8 +73,8 @@ func newPool(l *zap.Logger, v *viper.Viper) *Pool { currentIdx: atomic.NewInt32(-1), // fill with defaults: - requestTimeout: defaultRequestTimeout, - connectTimeout: defaultConnectTimeout, + reqTimeout: defaultRequestTimeout, + conTimeout: defaultConnectTimeout, opts: keepalive.ClientParameters{ Time: defaultKeepaliveTime, Timeout: defaultKeepaliveTimeout, @@ -90,12 +93,19 @@ func newPool(l *zap.Logger, v *viper.Viper) *Pool { rand.Seed(int64(seed)) l.Info("used random seed", zap.Uint64("seed", seed)) + p.reqHealth = new(state.HealthRequest) + p.reqHealth.SetTTL(service.NonForwardingTTL) + + if err := service.SignDataWithSessionToken(key, p.reqHealth); err != nil { + l.Fatal("could not sign `HealthRequest`", zap.Error(err)) + } + if val := v.GetDuration("conn_ttl"); val > 0 { p.ttl = val } if val := v.GetDuration("connect_timeout"); val > 0 { - p.connectTimeout = val + p.conTimeout = val } if val := v.GetDuration("keepalive.time"); val > 0 { @@ -149,29 +159,6 @@ func (p *Pool) close() { } } -func (a *app) checkHealth(ctx context.Context, conn *grpc.ClientConn) error { - //a.log.Info("try to fetch node health status", - // zap.String("node", conn.Target()), - // zap.Stringer("timeout", a.reqTimeout)) - - ctx, cancel := context.WithTimeout(ctx, a.reqTimeout) - result, err := state.NewStatusClient(conn).HealthCheck(ctx, a.reqHealth) - cancel() - - if err != nil { - result = &state.HealthResponse{Status: err.Error()} - } else if !result.Healthy { - err = errors.New(result.Status) - } - - a.log.Debug("received node health status", - zap.String("node", conn.Target()), - zap.String("status", result.Status), - zap.Error(err)) - - return err -} - func (p *Pool) reBalance(ctx context.Context) { p.Lock() defer func() { @@ -206,7 +193,7 @@ func (p *Pool) reBalance(ctx context.Context) { p.log.Warn("empty connection, try to connect", zap.String("address", p.nodes[i].address)) - ctx, cancel := context.WithTimeout(ctx, p.connectTimeout) + ctx, cancel := context.WithTimeout(ctx, p.conTimeout) conn, err = grpc.DialContext(ctx, p.nodes[i].address, grpc.WithBlock(), grpc.WithInsecure(), @@ -329,13 +316,10 @@ func (p *Pool) isAlive(ctx context.Context, cur *grpc.ClientConn) error { switch st := cur.GetState(); st { case connectivity.Idle, connectivity.Ready, connectivity.Connecting: - req := new(state.HealthRequest) - req.SetTTL(service.NonForwardingTTL) - - ctx, cancel := context.WithTimeout(ctx, p.requestTimeout) + ctx, cancel := context.WithTimeout(ctx, p.reqTimeout) defer cancel() - res, err := state.NewStatusClient(cur).HealthCheck(ctx, req) + res, err := state.NewStatusClient(cur).HealthCheck(ctx, p.reqHealth) if err != nil { p.log.Warn("could not fetch health-check", zap.Error(err)) From f2352f1e768e576bb163b59b77db32ad6a1dc17f Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 21 Aug 2020 02:50:14 +0300 Subject: [PATCH 076/548] change module name --- README.md | 10 +++++----- go.mod | 2 +- go_dev.mod | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c79c586..e6a02fa 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# NeoFS HTTP Gateway +# NeoFS HTTP Gate -NeoFS HTTP Gateway is example of tool that provides basic interactions with NeoFS. -You can download files from NeoFS Network using NeoFS Gateway. +NeoFS HTTP Gate is example of tool that provides basic interactions with NeoFS. +You can download files from NeoFS Network using NeoFS Gate. ## Install -```go get -u github.com/nspcc-dev/neofs-gw``` +```go get -u github.com/nspcc-dev/neofs-http-gate``` ## Configuration @@ -19,7 +19,7 @@ You can download files from NeoFS Network using NeoFS Gateway. --verbose debug gRPC connections --request_timeout duration gRPC request timeout (default 5s) --connect_timeout duration gRPC connect timeout (default 30s) - --listen_address string HTTP Gateway listen address (default "0.0.0.0:8082") + --listen_address string HTTP Gate listen address (default "0.0.0.0:8082") -p, --peers stringArray NeoFS nodes # Environments: diff --git a/go.mod b/go.mod index 8ed1c0a..549cced 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/nspcc-dev/neofs-gw +module github.com/nspcc-dev/neofs-http-gate go 1.13 diff --git a/go_dev.mod b/go_dev.mod index aaf657b..8077c1d 100644 --- a/go_dev.mod +++ b/go_dev.mod @@ -1,4 +1,4 @@ -module github.com/nspcc-dev/neofs-gw +module github.com/nspcc-dev/neofs-http-gate go 1.13 From 6226729e38b3011bdc95fa8d8aa5b82a04fa25c2 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 9 Nov 2020 16:43:23 +0300 Subject: [PATCH 077/548] Migrate to CDN SDK --- app.go | 119 ++++++++++++------- go.mod | 9 +- go.sum | 171 ++++++++++++++++++++------- grace.go | 24 ---- health.go | 7 +- logger.go | 143 ---------------------- main.go | 41 ++++++- pool.go | 335 ---------------------------------------------------- receive.go | 169 ++++++++------------------ settings.go | 34 +----- 10 files changed, 304 insertions(+), 748 deletions(-) delete mode 100644 grace.go delete mode 100644 logger.go delete mode 100644 pool.go diff --git a/app.go b/app.go index b255c9e..c3e9f1a 100644 --- a/app.go +++ b/app.go @@ -3,33 +3,39 @@ package main import ( "context" "crypto/ecdsa" - "errors" + "crypto/elliptic" + "crypto/rand" + "strconv" "time" "github.com/fasthttp/router" + sdk "github.com/nspcc-dev/cdn-neofs-sdk" + "github.com/nspcc-dev/cdn-neofs-sdk/creds/neofs" + "github.com/nspcc-dev/cdn-neofs-sdk/logger" + "github.com/nspcc-dev/cdn-neofs-sdk/pool" + crypto "github.com/nspcc-dev/neofs-crypto" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" + "google.golang.org/grpc" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/keepalive" ) type ( app struct { - pool *Pool + cli sdk.Client + pool pool.Client log *zap.Logger cfg *viper.Viper key *ecdsa.PrivateKey - wlog logger + wlog logger.Logger web *fasthttp.Server jobDone chan struct{} webDone chan struct{} - rebalanceTimer time.Duration - - nodes []string - conTimeout time.Duration reqTimeout time.Duration } @@ -61,7 +67,7 @@ func WithConfig(c *viper.Viper) Option { } } -func newApp(opt ...Option) App { +func newApp(ctx context.Context, opt ...Option) App { a := &app{ log: zap.L(), cfg: viper.GetViper(), @@ -75,14 +81,12 @@ func newApp(opt ...Option) App { opt[i](a) } - a.wlog = gRPCLogger(a.log) + a.wlog = logger.GRPC(a.log) if a.cfg.GetBool("verbose") { grpclog.SetLoggerV2(a.wlog) } - a.key = fetchKey(a.log, a.cfg) - a.rebalanceTimer = a.cfg.GetDuration("rebalance_timer") a.conTimeout = a.cfg.GetDuration("connect_timeout") a.reqTimeout = a.cfg.GetDuration("request_timeout") @@ -98,11 +102,68 @@ func newApp(opt ...Option) App { a.web.NoDefaultContentType = true // -- -- -- -- -- -- -- -- -- -- - a.pool = newPool(a.log, a.cfg, a.key) + connections := make(map[string]float64) + for i := 0; ; i++ { + address := a.cfg.GetString("peers." + strconv.Itoa(i) + ".address") + weight := a.cfg.GetFloat64("peers." + strconv.Itoa(i) + ".weight") + if address == "" { + break + } + + connections[address] = weight + } + + cred, err := prepareCredentials(a.cfg.GetString("key"), a.log) + if err != nil { + a.log.Fatal("could not prepare credentials", zap.Error(err)) + } + + a.pool, err = pool.New(ctx, + pool.WithLogger(a.log), + pool.WithCredentials(cred), + pool.WithWeightPool(connections), + pool.WithGRPCOptions( + grpc.WithBlock(), + grpc.WithInsecure(), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: a.cfg.GetDuration("keepalive.time"), + Timeout: a.cfg.GetDuration("keepalive.timeout"), + PermitWithoutStream: a.cfg.GetBool("keepalive.permit_without_stream"), + }))) + + if err != nil { + a.log.Fatal("could not prepare connection pool", zap.Error(err)) + } + + a.cli, err = sdk.New(ctx, + sdk.WithLogger(a.log), + sdk.WithCredentials(cred), + sdk.WithConnectionPool(a.pool)) + if err != nil { + a.log.Fatal("could not prepare sdk client", zap.Error(err)) + } return a } +func prepareCredentials(key string, log *zap.Logger) (neofs.Credentials, error) { + if key == generated { + sk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + + key, err = crypto.WIFEncode(sk) + if err != nil { + return nil, err + } + + log.Info("generate new key", zap.String("wif", key)) + } + + return neofs.New(key) +} + func (a *app) Wait() { a.log.Info("application started") @@ -115,37 +176,7 @@ func (a *app) Wait() { } func (a *app) Worker(ctx context.Context) { - dur := a.rebalanceTimer - tick := time.NewTimer(dur) - - a.pool.reBalance(ctx) - - switch _, err := a.pool.getConnection(ctx); { - case err == nil: - // ignore - case errors.Is(err, context.Canceled): - // ignore - // l.Info("context canceled") - default: - a.log.Fatal("could get connection", zap.Error(err)) - } - -loop: - for { - select { - case <-ctx.Done(): - break loop - case <-tick.C: - a.pool.reBalance(ctx) - tick.Reset(dur) - } - } - - a.pool.close() - tick.Stop() - - a.log.Info("connection worker stopped") - + a.pool.Worker(ctx) close(a.jobDone) } @@ -163,7 +194,7 @@ func (a *app) Serve(ctx context.Context) { r.GET("/get/{cid}/{oid}", a.receiveFile) // attaching /-/(ready,healthy) - attachHealthy(r, a.pool.unhealthy) + attachHealthy(r, a.pool.Status) // enable metrics if a.cfg.GetBool("metrics") { diff --git a/go.mod b/go.mod index 549cced..6c39651 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,8 @@ go 1.13 require ( github.com/fasthttp/router v1.1.6 - github.com/nspcc-dev/neofs-api-go v1.0.0 + github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201110112919-ed1430a93074 + github.com/nspcc-dev/neofs-api-go v0.0.0-20201029071528-352e99d9b91a github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.6.0 @@ -12,8 +13,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.0 github.com/valyala/fasthttp v1.14.0 - go.uber.org/atomic v1.6.0 - go.uber.org/zap v1.15.0 - golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 // indirect - google.golang.org/grpc v1.29.1 + go.uber.org/zap v1.16.0 + google.golang.org/grpc v1.33.1 ) diff --git a/go.sum b/go.sum index 70d1486..8681387 100644 --- a/go.sum +++ b/go.sum @@ -14,52 +14,82 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1:MJCkWUBhi9pn/CrYO1Q3P687y2KeahrOPS9BD9LDGb0= +github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= +github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= +github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310 h1:t+qxRrRtwNiUYA+Xh2jSXhoG2grnMCMKX4Fg6lx9X1U= -github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp/router v1.1.6 h1:lBcXxp1ZNoNbSeh4+RvAaXKSEiHU6sGd+gEMpd5Xjog= github.com/fasthttp/router v1.1.6/go.mod h1:E1mpv7mrQzAhiSQdqhRb+GBTC7MEV+bLFVmgzSA5oFM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= +github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -68,13 +98,11 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -83,16 +111,17 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -105,8 +134,9 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -136,35 +166,47 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.10.4 h1:jFzIFaf586tquEB5EhzQG0HwGNSlgAJpG53G6Ss11wc= github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= @@ -175,26 +217,41 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= -github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= -github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neofs-api-go v1.0.0 h1:FSsV3UFjTfBjaqgaOW7pZ9J8wqg4uOEF+DGfH04sY/M= -github.com/nspcc-dev/neofs-api-go v1.0.0/go.mod h1:2tf31g2Ns/Z2ev5d8LZ/9f1VHIeY5LHpDbq4EsDhYM0= +github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201110112919-ed1430a93074 h1:taQ+D9bzhnLMMdHFLRZCT537mq1P8/qfZrO1qaino0o= +github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201110112919-ed1430a93074/go.mod h1:p/YXjSDO2pJ5JwYKrLwVKIxdcVStjFxD0gfzIrwy5eY= +github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= +github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= +github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= +github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= +github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E= +github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= +github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= +github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= +github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= +github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= +github.com/nspcc-dev/neofs-api-go v0.0.0-20201029071528-352e99d9b91a h1:A/8D9A9Fbp4tw1xGgfZB0vtPKEQ/LYq0l7DFuM4KeS4= +github.com/nspcc-dev/neofs-api-go v0.0.0-20201029071528-352e99d9b91a/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= +github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/netmap v1.7.0 h1:ak64xn/gPdgYw4tsqSSF7kAGQGbEpeuJEF3XwBX4L9Y= -github.com/nspcc-dev/netmap v1.7.0/go.mod h1:mhV3UOg9ljQmu0teQShD6+JYX09XY5gu2I4hIByCH9M= +github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= -github.com/nspcc-dev/tzhash v1.4.0 h1:RVIR+mxOBHl58CE99+DXtE31ylD5PEkZSoWqoj4fVjg= -github.com/nspcc-dev/tzhash v1.4.0/go.mod h1:Z8gp/VZbyWgPhaMp/KTmeoW5UTynp/N60g0jTtSzBws= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -205,6 +262,7 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -215,18 +273,20 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/savsgio/gotils v0.0.0-20200413113635-8c468ce75cca h1:Qe7Mtuhjkk38HVpRtvWdziZJcwG3Qup1mfyvyOrcnyM= github.com/savsgio/gotils v0.0.0-20200413113635-8c468ce75cca/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= @@ -245,31 +305,40 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.14.0 h1:67bfuW9azCMwW/Jlq/C+VeihNpAuJMWkYPBig1gdi3A= github.com/valyala/fasthttp v1.14.0/go.mod h1:ol1PCaL0dX20wC0htZ7sYCsvCYmrouYra0zHzaclZhE= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= +github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -281,16 +350,19 @@ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKY go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc= -golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -310,8 +382,10 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -325,8 +399,9 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -339,11 +414,14 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -351,11 +429,16 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -363,8 +446,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -381,11 +464,12 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200123022218-593de606220b h1:ztSlcncMErSAUzXwnVO1iTPxHwtvOHBB26SGiyYXIEE= +golang.org/x/tools v0.0.0-20200123022218-593de606220b/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -403,7 +487,6 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= @@ -413,13 +496,13 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.33.1 h1:DGeFlSan2f+WEtCERJ4J9GJWk15TxUi8QGagfI87Xyc= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= @@ -430,15 +513,19 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/grace.go b/grace.go deleted file mode 100644 index 8469d62..0000000 --- a/grace.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "context" - "os" - "os/signal" - "syscall" - - "go.uber.org/zap" -) - -// newGracefulContext returns graceful context -func newGracefulContext(l *zap.Logger) context.Context { - ctx, cancel := context.WithCancel(context.Background()) - go func() { - ch := make(chan os.Signal, 1) - signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) - sig := <-ch - l.Info("received signal", - zap.String("signal", sig.String())) - cancel() - }() - return ctx -} diff --git a/health.go b/health.go index 3141a0f..4483e70 100644 --- a/health.go +++ b/health.go @@ -3,15 +3,16 @@ package main import ( "github.com/fasthttp/router" "github.com/valyala/fasthttp" - "go.uber.org/atomic" ) +type stater func() error + const ( healthyState = "NeoFS HTTP Gateway is " defaultContentType = "text/plain; charset=utf-8" ) -func attachHealthy(r *router.Router, e *atomic.Error) { +func attachHealthy(r *router.Router, e stater) { r.GET("/-/ready/", func(ctx *fasthttp.RequestCtx) { ctx.SetStatusCode(fasthttp.StatusOK) ctx.SetBodyString(healthyState + "ready") @@ -21,7 +22,7 @@ func attachHealthy(r *router.Router, e *atomic.Error) { code := fasthttp.StatusOK msg := "healthy" - if err := e.Load(); err != nil { + if err := e(); err != nil { msg = "unhealthy: " + err.Error() code = fasthttp.StatusBadRequest } diff --git a/logger.go b/logger.go deleted file mode 100644 index 3bb969e..0000000 --- a/logger.go +++ /dev/null @@ -1,143 +0,0 @@ -package main - -import ( - "strings" - - "google.golang.org/grpc/grpclog" - - "github.com/spf13/viper" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -type ( - zapLogger struct { - zapcore.Core - log *zap.SugaredLogger - } - - logger interface { - grpclog.LoggerV2 - Println(v ...interface{}) - } -) - -const ( - formatJSON = "json" - formatConsole = "console" - - defaultSamplingInitial = 100 - defaultSamplingThereafter = 100 -) - -func gRPCLogger(l *zap.Logger) logger { - log := l.WithOptions( - // skip gRPCLog + zapLogger in caller - zap.AddCallerSkip(2)) - return &zapLogger{ - Core: log.Core(), - log: log.Sugar(), - } -} - -func safeLevel(lvl string) zap.AtomicLevel { - switch strings.ToLower(lvl) { - case "debug": - return zap.NewAtomicLevelAt(zap.DebugLevel) - case "warn": - return zap.NewAtomicLevelAt(zap.WarnLevel) - case "error": - return zap.NewAtomicLevelAt(zap.ErrorLevel) - case "fatal": - return zap.NewAtomicLevelAt(zap.FatalLevel) - case "panic": - return zap.NewAtomicLevelAt(zap.PanicLevel) - default: - return zap.NewAtomicLevelAt(zap.InfoLevel) - } -} - -func newLogger(v *viper.Viper) *zap.Logger { - c := zap.NewProductionConfig() - - c.OutputPaths = []string{"stdout"} - c.ErrorOutputPaths = []string{"stdout"} - - if v.IsSet("logger.sampling") { - c.Sampling = &zap.SamplingConfig{ - Initial: defaultSamplingInitial, - Thereafter: defaultSamplingThereafter, - } - - if val := v.GetInt("logger.sampling.initial"); val > 0 { - c.Sampling.Initial = val - } - - if val := v.GetInt("logger.sampling.thereafter"); val > 0 { - c.Sampling.Thereafter = val - } - } - - // logger level - c.Level = safeLevel(v.GetString("logger.level")) - traceLvl := safeLevel(v.GetString("logger.trace_level")) - - // logger format - switch f := v.GetString("logger.format"); strings.ToLower(f) { - case formatConsole: - c.Encoding = formatConsole - default: - c.Encoding = formatJSON - } - - // logger time - c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - - l, err := c.Build( - // enable trace only for current log-level - zap.AddStacktrace(traceLvl)) - if err != nil { - panic(err) - } - - if v.GetBool("logger.no_disclaimer") { - return l - } - - name := v.GetString("app.name") - version := v.GetString("app.version") - - return l.With( - zap.String("app_name", name), - zap.String("app_version", version)) -} - -func (z *zapLogger) Info(args ...interface{}) { z.log.Info(args...) } - -func (z *zapLogger) Infoln(args ...interface{}) { z.log.Info(args...) } - -func (z *zapLogger) Infof(format string, args ...interface{}) { z.log.Infof(format, args...) } - -func (z *zapLogger) Println(args ...interface{}) { z.log.Info(args...) } - -func (z *zapLogger) Printf(format string, args ...interface{}) { z.log.Infof(format, args...) } - -func (z *zapLogger) Warning(args ...interface{}) { z.log.Warn(args...) } - -func (z *zapLogger) Warningln(args ...interface{}) { z.log.Warn(args...) } - -func (z *zapLogger) Warningf(format string, args ...interface{}) { z.log.Warnf(format, args...) } - -func (z *zapLogger) Error(args ...interface{}) { z.log.Error(args...) } - -func (z *zapLogger) Errorln(args ...interface{}) { z.log.Error(args...) } - -func (z *zapLogger) Errorf(format string, args ...interface{}) { z.log.Errorf(format, args...) } - -func (z *zapLogger) Fatal(args ...interface{}) { z.log.Fatal(args...) } - -func (z *zapLogger) Fatalln(args ...interface{}) { z.log.Fatal(args...) } - -func (z *zapLogger) Fatalf(format string, args ...interface{}) { z.Fatalf(format, args...) } - -func (z *zapLogger) V(int) bool { return z.Enabled(zapcore.DebugLevel) } diff --git a/main.go b/main.go index 1d9c832..36c12f1 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,49 @@ package main +import ( + "github.com/nspcc-dev/cdn-neofs-sdk/grace" + "github.com/nspcc-dev/cdn-neofs-sdk/logger" + "github.com/spf13/viper" + "go.uber.org/zap" +) + +func newLogger(v *viper.Viper) *zap.Logger { + options := []logger.Option{ + logger.WithLevel(v.GetString("logger.level")), + logger.WithTraceLevel(v.GetString("logger.trace_level")), + + logger.WithFormat(v.GetString("logger.format")), + + logger.WithSamplingInitial(v.GetInt("logger.sampling.initial")), + logger.WithSamplingThereafter(v.GetInt("logger.sampling.thereafter")), + + logger.WithAppName(v.GetString("app_name")), + logger.WithAppVersion(v.GetString("app_version")), + } + + if v.GetBool("logger.no_caller") { + options = append(options, logger.WithoutCaller()) + } + + if v.GetBool("logger.no_disclaimer") { + options = append(options, logger.WithoutDisclaimer()) + } + + l, err := logger.New(options...) + if err != nil { + panic(err) + } + + return l +} + func main() { var ( v = settings() l = newLogger(v) - g = newGracefulContext(l) + g = grace.Context(l) - a = newApp( + a = newApp(g, WithLogger(l), WithConfig(v)) ) diff --git a/pool.go b/pool.go deleted file mode 100644 index d45cd44..0000000 --- a/pool.go +++ /dev/null @@ -1,335 +0,0 @@ -package main - -import ( - "context" - "crypto/ecdsa" - crand "crypto/rand" - "encoding/binary" - "errors" - "math/rand" - "sort" - "strconv" - "sync" - "time" - - "github.com/nspcc-dev/neofs-api-go/service" - "github.com/nspcc-dev/neofs-api-go/state" - "github.com/spf13/viper" - "go.uber.org/atomic" - "go.uber.org/zap" - "google.golang.org/grpc" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/keepalive" -) - -type ( - node struct { - index int32 - address string - weight uint32 - usedAt time.Time - conn *grpc.ClientConn - } - - Pool struct { - log *zap.Logger - - ttl time.Duration - - conTimeout time.Duration - reqTimeout time.Duration - opts keepalive.ClientParameters - - reqHealth *state.HealthRequest - - currentIdx *atomic.Int32 - currentConn *grpc.ClientConn - - *sync.Mutex - nodes []*node - keys []uint32 - conns map[uint32][]*node - - unhealthy *atomic.Error - } -) - -var ( - errBootstrapping = errors.New("bootstrapping") - errEmptyConnection = errors.New("empty connection") - errNoHealthyConnections = errors.New("no active connections") -) - -func newPool(l *zap.Logger, v *viper.Viper, key *ecdsa.PrivateKey) *Pool { - p := &Pool{ - log: l, - Mutex: new(sync.Mutex), - keys: make([]uint32, 0), - nodes: make([]*node, 0), - conns: make(map[uint32][]*node), - - ttl: defaultTTL, - - currentIdx: atomic.NewInt32(-1), - - // fill with defaults: - reqTimeout: defaultRequestTimeout, - conTimeout: defaultConnectTimeout, - opts: keepalive.ClientParameters{ - Time: defaultKeepaliveTime, - Timeout: defaultKeepaliveTimeout, - PermitWithoutStream: true, - }, - - unhealthy: atomic.NewError(errBootstrapping), - } - - buf := make([]byte, 8) - if _, err := crand.Read(buf); err != nil { - l.Panic("could not read seed", zap.Error(err)) - } - - seed := binary.BigEndian.Uint64(buf) - rand.Seed(int64(seed)) - l.Info("used random seed", zap.Uint64("seed", seed)) - - p.reqHealth = new(state.HealthRequest) - p.reqHealth.SetTTL(service.NonForwardingTTL) - - if err := service.SignDataWithSessionToken(key, p.reqHealth); err != nil { - l.Fatal("could not sign `HealthRequest`", zap.Error(err)) - } - - if val := v.GetDuration("conn_ttl"); val > 0 { - p.ttl = val - } - - if val := v.GetDuration("connect_timeout"); val > 0 { - p.conTimeout = val - } - - if val := v.GetDuration("keepalive.time"); val > 0 { - p.opts.Time = val - } - - if val := v.GetDuration("keepalive.timeout"); val > 0 { - p.opts.Timeout = val - } - - if v.IsSet("keepalive.permit_without_stream") { - p.opts.PermitWithoutStream = v.GetBool("keepalive.permit_without_stream") - } - - for i := 0; ; i++ { - key := "peers." + strconv.Itoa(i) + "." - address := v.GetString(key + "address") - weight := v.GetFloat64(key + "weight") - - if address == "" { - l.Warn("skip, empty address") - break - } - - p.nodes = append(p.nodes, &node{ - index: int32(i), - address: address, - weight: uint32(weight * 100), - }) - - l.Info("add new peer", - zap.String("address", p.nodes[i].address), - zap.Uint32("weight", p.nodes[i].weight)) - } - - return p -} - -func (p *Pool) close() { - p.Lock() - defer p.Unlock() - - for i := range p.nodes { - if p.nodes[i] == nil || p.nodes[i].conn == nil { - continue - } - - p.log.Warn("close connection", - zap.String("address", p.nodes[i].address), - zap.Error(p.nodes[i].conn.Close())) - } -} - -func (p *Pool) reBalance(ctx context.Context) { - p.Lock() - defer func() { - p.Unlock() - - _, err := p.getConnection(ctx) - p.unhealthy.Store(err) - }() - - keys := make(map[uint32]struct{}) - - p.log.Info("re-balancing connections") - - for i := range p.nodes { - var ( - idx = -1 - exists bool - err error - start = time.Now() - conn = p.nodes[i].conn - weight = p.nodes[i].weight - ) - - if err = ctx.Err(); err != nil { - p.log.Warn("something went wrong", zap.Error(err)) - p.unhealthy.Store(err) - - return - } - - if conn == nil { - p.log.Warn("empty connection, try to connect", - zap.String("address", p.nodes[i].address)) - - ctx, cancel := context.WithTimeout(ctx, p.conTimeout) - conn, err = grpc.DialContext(ctx, p.nodes[i].address, - grpc.WithBlock(), - grpc.WithInsecure(), - grpc.WithKeepaliveParams(p.opts)) - cancel() - - if err != nil || conn == nil { - p.log.Warn("skip, could not connect to node", - zap.String("address", p.nodes[i].address), - zap.Stringer("elapsed", time.Since(start)), - zap.Error(err)) - continue - } - - p.nodes[i].conn = conn - p.nodes[i].usedAt = time.Now() - p.log.Info("connected to node", zap.String("address", p.nodes[i].address)) - } - - for j := range p.conns[weight] { - if p.conns[weight][j] != nil && p.conns[weight][j].conn == conn { - idx = j - exists = true - break - } - } - - usedAt := time.Since(p.nodes[i].usedAt) - - // if something wrong with connection (bad state, unhealthy or not used a long time), try to close it and remove - if err = p.isAlive(ctx, conn); err != nil || usedAt > p.ttl { - p.log.Warn("connection not alive", - zap.String("address", p.nodes[i].address), - zap.Stringer("used_at", usedAt), - zap.Error(err)) - - if exists { - // remove from connections - p.conns[weight] = append(p.conns[weight][:idx], p.conns[weight][idx+1:]...) - } - - if err = conn.Close(); err != nil { - p.log.Warn("could not close bad connection", - zap.String("address", p.nodes[i].address), - zap.Stringer("used_at", usedAt), - zap.Error(err)) - } - - if p.nodes[i].conn != nil { - p.nodes[i].conn = nil - } - continue - } - - keys[weight] = struct{}{} - - p.log.Info("connection alive", - zap.String("address", p.nodes[i].address), - zap.Stringer("used_at", usedAt)) - - if !exists { - p.conns[weight] = append(p.conns[weight], p.nodes[i]) - } - } - - p.keys = p.keys[:0] - for w := range keys { - p.keys = append(p.keys, w) - } - - sort.Slice(p.keys, func(i, j int) bool { - return p.keys[i] > p.keys[j] - }) -} - -func (p *Pool) getConnection(ctx context.Context) (*grpc.ClientConn, error) { - p.Lock() - defer p.Unlock() - - if err := p.isAlive(ctx, p.currentConn); err == nil { - if id := p.currentIdx.Load(); id != -1 && p.nodes[id] != nil { - p.nodes[id].usedAt = time.Now() - } - - return p.currentConn, nil - } - - for _, w := range p.keys { - switch ln := len(p.conns[w]); ln { - case 0: - continue - case 1: - p.currentConn = p.conns[w][0].conn - p.conns[w][0].usedAt = time.Now() - p.currentIdx.Store(p.conns[w][0].index) - return p.currentConn, nil - default: // > 1 - i := rand.Intn(ln) - p.currentConn = p.conns[w][i].conn - p.conns[w][i].usedAt = time.Now() - p.currentIdx.Store(p.conns[w][i].index) - return p.currentConn, nil - } - } - - p.currentConn = nil - p.currentIdx.Store(-1) - - if ctx.Err() != nil { - return nil, ctx.Err() - } - - return nil, errNoHealthyConnections -} - -func (p *Pool) isAlive(ctx context.Context, cur *grpc.ClientConn) error { - if cur == nil { - return errEmptyConnection - } - - switch st := cur.GetState(); st { - case connectivity.Idle, connectivity.Ready, connectivity.Connecting: - ctx, cancel := context.WithTimeout(ctx, p.reqTimeout) - defer cancel() - - res, err := state.NewStatusClient(cur).HealthCheck(ctx, p.reqHealth) - if err != nil { - p.log.Warn("could not fetch health-check", zap.Error(err)) - - return err - } else if !res.Healthy { - return errors.New(res.Status) - } - - return nil - default: - return errors.New(st.String()) - } -} diff --git a/receive.go b/receive.go index 58109e6..9f8a8a5 100644 --- a/receive.go +++ b/receive.go @@ -1,35 +1,53 @@ package main import ( - "context" "io" "net/http" "path" "strconv" + "strings" + "sync" "time" - "github.com/nspcc-dev/neofs-api-go/object" - "github.com/nspcc-dev/neofs-api-go/refs" - "github.com/nspcc-dev/neofs-api-go/service" + sdk "github.com/nspcc-dev/cdn-neofs-sdk" + "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/pkg/errors" "github.com/valyala/fasthttp" "go.uber.org/zap" - "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) +type detector struct { + io.Writer + sync.Once + + contentType string +} + +func newDetector(w io.Writer) *detector { + return &detector{Writer: w} +} + +func (d *detector) Write(data []byte) (int, error) { + d.Once.Do(func() { + d.contentType = http.DetectContentType(data) + }) + + return d.Writer.Write(data) +} + func (a *app) receiveFile(c *fasthttp.RequestCtx) { var ( err error - cid refs.CID - oid refs.ObjectID + disp = "inline" start = time.Now() - con *grpc.ClientConn - cli object.Service_GetClient - ctx context.Context + address = object.NewAddress() sCID, _ = c.UserValue("cid").(string) sOID, _ = c.UserValue("oid").(string) + value = strings.Join([]string{sCID, sOID}, "/") + + filename string ) log := a.log.With( @@ -37,57 +55,15 @@ func (a *app) receiveFile(c *fasthttp.RequestCtx) { zap.String("cid", sCID), zap.String("oid", sOID)) - if err = cid.Parse(sCID); err != nil { - log.Error("wrong container id", zap.Error(err)) - - c.Error("wrong container id", fasthttp.StatusBadRequest) - return - } else if err = oid.Parse(sOID); err != nil { - log.Error("wrong object id", zap.Error(err)) - - c.Error("wrong object id", fasthttp.StatusBadRequest) + if err = address.Parse(value); err != nil { + log.Error("wrong object address", zap.Error(err)) + c.Error("wrong object address", fasthttp.StatusBadRequest) return } - { // try to connect or throw http error: - ctx, cancel := context.WithTimeout(c, a.reqTimeout) - defer cancel() - - if con, err = a.pool.getConnection(ctx); err != nil { - log.Error("getConnection timeout", zap.Error(err)) - c.Error("could not get alive connection", fasthttp.StatusBadRequest) - return - } - } - - ctx, cancel := context.WithTimeout(c, a.reqTimeout) - defer cancel() - - log = log.With(zap.String("node", con.Target())) - - defer func() { - if err != nil { - return - } - - log.Info("object sent to client", zap.Stringer("elapsed", time.Since(start))) - }() - - req := &object.GetRequest{Address: refs.Address{ObjectID: oid, CID: cid}} - req.SetTTL(service.SingleForwardingTTL) - - if err = service.SignDataWithSessionToken(a.key, req); err != nil { - log.Error("could not sign request", zap.Error(err)) - c.Error("could not sign request", fasthttp.StatusBadRequest) - return - } - - if cli, err = object.NewServiceClient(con).Get(ctx, req); err != nil { - log.Error("could not prepare connection", zap.Error(err)) - - c.Error("could not prepare connection", fasthttp.StatusBadRequest) - return - } else if err = receiveObject(c, cli); err != nil { + writer := newDetector(c.Response.BodyWriter()) + obj, err := a.cli.Object().Get(c, address, sdk.WithGetWriter(writer)) + if err != nil { log.Error("could not receive object", zap.Stringer("elapsed", time.Since(start)), zap.Error(err)) @@ -108,67 +84,26 @@ func (a *app) receiveFile(c *fasthttp.RequestCtx) { c.Error(msg, code) return } -} -func receiveObject(c *fasthttp.RequestCtx, cli object.Service_GetClient) error { - var ( - typ string - content_disp_type string = "inline" - put = c.Request.URI().QueryArgs().GetBool("download") - ) - - for { - resp, err := cli.Recv() - if err != nil { - if err == io.EOF { - break - } - return err - } - - switch o := resp.R.(type) { - case *object.GetResponse_Object: - obj := o.Object - - c.Response.Header.Set("Content-Length", strconv.FormatUint(obj.SystemHeader.PayloadLength, 10)) - c.Response.Header.Set("x-object-id", obj.SystemHeader.ID.String()) - c.Response.Header.Set("x-owner-id", obj.SystemHeader.OwnerID.String()) - c.Response.Header.Set("x-container-id", obj.SystemHeader.CID.String()) - - for i := range obj.Headers { - if hdr := obj.Headers[i].GetUserHeader(); hdr != nil { - c.Response.Header.Set("x-"+hdr.Key, hdr.Value) - - if hdr.Key == object.FilenameHeader { - if put { - content_disp_type = "attachment" - } - // NOTE: we use path.Base because hdr.Value can be something like `/path/to/filename.ext` - c.Response.Header.Set("Content-Disposition", content_disp_type+"; filename="+path.Base(hdr.Value)) - } - } - } - - if len(obj.Payload) > 0 { - typ = http.DetectContentType(obj.Payload) - } - - if _, err = c.Write(obj.Payload); err != nil { - return err - } - - case *object.GetResponse_Chunk: - if typ == "" { - typ = http.DetectContentType(o.Chunk) - } - - if _, err = c.Write(o.Chunk); err != nil { - return err - } - } + if c.Request.URI().QueryArgs().GetBool("download") { + disp = "attachment" } - c.SetContentType(typ) + c.Response.Header.Set("Content-Length", strconv.FormatUint(obj.GetPayloadSize(), 10)) + c.Response.Header.Set("x-object-id", obj.GetID().String()) + c.Response.Header.Set("x-owner-id", obj.GetOwnerID().String()) + c.Response.Header.Set("x-container-id", obj.GetContainerID().String()) - return nil + for _, attr := range obj.GetAttributes() { + key := attr.GetKey() + val := attr.GetValue() + if key == object.AttributeFileName { + filename = val + } + + c.Response.Header.Set("x-"+key, val) + } + + c.SetContentType(writer.contentType) + c.Response.Header.Set("Content-Disposition", disp+"; filename="+path.Base(filename)) } diff --git a/settings.go b/settings.go index d20e54c..3a413e4 100644 --- a/settings.go +++ b/settings.go @@ -1,9 +1,6 @@ package main import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" "fmt" "io" "os" @@ -11,11 +8,8 @@ import ( "strings" "time" - "github.com/nspcc-dev/neofs-api-go/refs" - crypto "github.com/nspcc-dev/neofs-crypto" "github.com/spf13/pflag" "github.com/spf13/viper" - "go.uber.org/zap" ) type empty int @@ -38,33 +32,6 @@ const ( func (empty) Read([]byte) (int, error) { return 0, io.EOF } -func fetchKey(l *zap.Logger, v *viper.Viper) *ecdsa.PrivateKey { - switch val := v.GetString("key"); val { - case generated: - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - l.Fatal("could not generate private key", zap.Error(err)) - } - - id, err := refs.NewOwnerID(&key.PublicKey) - l.Info("generate new key", - zap.Stringer("key", id), - zap.Error(err)) - - return key - - default: - key, err := crypto.LoadPrivateKey(val) - if err != nil { - l.Fatal("could not load private key", - zap.String("key", v.GetString("key")), - zap.Error(err)) - } - - return key - } -} - func settings() *viper.Viper { v := viper.New() v.AutomaticEnv() @@ -104,6 +71,7 @@ func settings() *viper.Viper { v.SetDefault("logger.level", "debug") v.SetDefault("logger.format", "console") v.SetDefault("logger.trace_level", "fatal") + v.SetDefault("logger.no_caller", false) v.SetDefault("logger.no_disclaimer", true) v.SetDefault("logger.sampling.initial", 1000) v.SetDefault("logger.sampling.thereafter", 1000) From f14e2b695edce8b5b176ce8c9fb054ec919132f1 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 13 Nov 2020 21:01:10 +0300 Subject: [PATCH 078/548] Fixed NPE in CDN SDK --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6c39651..fea2106 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/fasthttp/router v1.1.6 - github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201110112919-ed1430a93074 + github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201113175725-f448ffe8047c github.com/nspcc-dev/neofs-api-go v0.0.0-20201029071528-352e99d9b91a github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 8681387..a21dcc6 100644 --- a/go.sum +++ b/go.sum @@ -220,8 +220,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201110112919-ed1430a93074 h1:taQ+D9bzhnLMMdHFLRZCT537mq1P8/qfZrO1qaino0o= -github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201110112919-ed1430a93074/go.mod h1:p/YXjSDO2pJ5JwYKrLwVKIxdcVStjFxD0gfzIrwy5eY= +github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201113175725-f448ffe8047c h1:la/q4996O/wKifXCFWQ4WKOxnQLVtaEeGJBVlQ/AUyY= +github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201113175725-f448ffe8047c/go.mod h1:p/YXjSDO2pJ5JwYKrLwVKIxdcVStjFxD0gfzIrwy5eY= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= From a4f7a2f9627181aa83c188db9a95a0a1fcc397e3 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 23 Nov 2020 12:32:03 +0300 Subject: [PATCH 079/548] Migrate to new release of NeoFS API - migrate to new release NeoFS API / CDN SDK - add `Last-Modified` header Signed-off-by: Evgeniy Kulikov --- app.go | 21 ++++++++++++++------- go.mod | 6 ++++-- go.sum | 6 ++---- receive.go | 35 +++++++++++++++++++++++++---------- 4 files changed, 45 insertions(+), 23 deletions(-) diff --git a/app.go b/app.go index c3e9f1a..b40ad53 100644 --- a/app.go +++ b/app.go @@ -6,7 +6,6 @@ import ( "crypto/elliptic" "crypto/rand" "strconv" - "time" "github.com/fasthttp/router" sdk "github.com/nspcc-dev/cdn-neofs-sdk" @@ -35,9 +34,6 @@ type ( jobDone chan struct{} webDone chan struct{} - - conTimeout time.Duration - reqTimeout time.Duration } App interface { @@ -87,8 +83,9 @@ func newApp(ctx context.Context, opt ...Option) App { grpclog.SetLoggerV2(a.wlog) } - a.conTimeout = a.cfg.GetDuration("connect_timeout") - a.reqTimeout = a.cfg.GetDuration("request_timeout") + conTimeout := a.cfg.GetDuration("connect_timeout") + reqTimeout := a.cfg.GetDuration("request_timeout") + tckTimeout := a.cfg.GetDuration("rebalance_timer") // -- setup FastHTTP server: -- a.web.Name = "neofs-http-gate" @@ -111,6 +108,9 @@ func newApp(ctx context.Context, opt ...Option) App { } connections[address] = weight + a.log.Info("add connection peer", + zap.String("address", address), + zap.Float64("weight", weight)) } cred, err := prepareCredentials(a.cfg.GetString("key"), a.log) @@ -122,6 +122,10 @@ func newApp(ctx context.Context, opt ...Option) App { pool.WithLogger(a.log), pool.WithCredentials(cred), pool.WithWeightPool(connections), + pool.WithTickerTimeout(tckTimeout), + pool.WithConnectTimeout(conTimeout), + pool.WithRequestTimeout(reqTimeout), + pool.WithAPIPreparer(sdk.APIPreparer), pool.WithGRPCOptions( grpc.WithBlock(), grpc.WithInsecure(), @@ -138,7 +142,8 @@ func newApp(ctx context.Context, opt ...Option) App { a.cli, err = sdk.New(ctx, sdk.WithLogger(a.log), sdk.WithCredentials(cred), - sdk.WithConnectionPool(a.pool)) + sdk.WithConnectionPool(a.pool), + sdk.WithAPIPreparer(sdk.APIPreparer)) if err != nil { a.log.Fatal("could not prepare sdk client", zap.Error(err)) } @@ -148,6 +153,8 @@ func newApp(ctx context.Context, opt ...Option) App { func prepareCredentials(key string, log *zap.Logger) (neofs.Credentials, error) { if key == generated { + log.Fatal("Don't use generated key, deprecated") + sk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { return nil, err diff --git a/go.mod b/go.mod index fea2106..dfedabe 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.13 require ( github.com/fasthttp/router v1.1.6 - github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201113175725-f448ffe8047c - github.com/nspcc-dev/neofs-api-go v0.0.0-20201029071528-352e99d9b91a + github.com/nspcc-dev/cdn-neofs-sdk v0.0.0 + github.com/nspcc-dev/neofs-api-go v1.20.2 github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.6.0 @@ -16,3 +16,5 @@ require ( go.uber.org/zap v1.16.0 google.golang.org/grpc v1.33.1 ) + +replace github.com/nspcc-dev/cdn-neofs-sdk => ../sdk diff --git a/go.sum b/go.sum index a21dcc6..71c40b0 100644 --- a/go.sum +++ b/go.sum @@ -220,8 +220,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201113175725-f448ffe8047c h1:la/q4996O/wKifXCFWQ4WKOxnQLVtaEeGJBVlQ/AUyY= -github.com/nspcc-dev/cdn-neofs-sdk v0.0.0-20201113175725-f448ffe8047c/go.mod h1:p/YXjSDO2pJ5JwYKrLwVKIxdcVStjFxD0gfzIrwy5eY= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= @@ -232,8 +230,8 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neofs-api-go v0.0.0-20201029071528-352e99d9b91a h1:A/8D9A9Fbp4tw1xGgfZB0vtPKEQ/LYq0l7DFuM4KeS4= -github.com/nspcc-dev/neofs-api-go v0.0.0-20201029071528-352e99d9b91a/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-api-go v1.20.2 h1:jOsfgOQdKhg7gTAGciRdIdOJxYX72CxqgkwaqzEscHw= +github.com/nspcc-dev/neofs-api-go v1.20.2/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= diff --git a/receive.go b/receive.go index 9f8a8a5..5688b0f 100644 --- a/receive.go +++ b/receive.go @@ -89,19 +89,34 @@ func (a *app) receiveFile(c *fasthttp.RequestCtx) { disp = "attachment" } - c.Response.Header.Set("Content-Length", strconv.FormatUint(obj.GetPayloadSize(), 10)) - c.Response.Header.Set("x-object-id", obj.GetID().String()) - c.Response.Header.Set("x-owner-id", obj.GetOwnerID().String()) - c.Response.Header.Set("x-container-id", obj.GetContainerID().String()) + c.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) + c.Response.Header.Set("x-object-id", obj.ID().String()) + c.Response.Header.Set("x-owner-id", obj.OwnerID().String()) + c.Response.Header.Set("x-container-id", obj.ContainerID().String()) - for _, attr := range obj.GetAttributes() { - key := attr.GetKey() - val := attr.GetValue() - if key == object.AttributeFileName { - filename = val - } + for _, attr := range obj.Attributes() { + key := attr.Key() + val := attr.Value() c.Response.Header.Set("x-"+key, val) + + switch key { + case object.AttributeFileName: + filename = val + case object.AttributeTimestamp: + value, err := strconv.ParseInt(val, 10, 64) + if err != nil { + a.log.Info("couldn't parse creation date", + zap.String("key", key), + zap.String("val", val), + zap.Error(err)) + continue + } + + c.Response.Header.Set("Last-Modified", + time.Unix(value, 0).Format(time.RFC1123)) + } + } c.SetContentType(writer.contentType) From 3166adc4108459d0e923ffab0324f76bab4bde5e Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 23 Nov 2020 12:43:27 +0300 Subject: [PATCH 080/548] Deprecation warning --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index e6a02fa..7b63d14 100644 --- a/README.md +++ b/README.md @@ -39,3 +39,6 @@ of Timeout and if no activity is seen even after that the connection is closed GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=Bool - if true, client sends keepalive pings even with no active RPCs. If false, when there are no active RPCs, Time and Timeout will be ignored and no keepalive pings will be sent. ``` + +### WARNING +`generated` value for `GW_KEY` or `--key` is deprecated, you should use pre-generated keys. \ No newline at end of file From f75eb4b80381c6ef6708906f37f35bc3a74863a6 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 27 Nov 2020 16:38:58 +0300 Subject: [PATCH 081/548] Update dependencies Signed-off-by: Evgeniy Kulikov --- go.mod | 14 ++-- go.sum | 207 +++++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 194 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index dfedabe..c85777c 100644 --- a/go.mod +++ b/go.mod @@ -3,18 +3,18 @@ module github.com/nspcc-dev/neofs-http-gate go 1.13 require ( - github.com/fasthttp/router v1.1.6 + github.com/fasthttp/router v1.3.3 github.com/nspcc-dev/cdn-neofs-sdk v0.0.0 - github.com/nspcc-dev/neofs-api-go v1.20.2 + github.com/nspcc-dev/neofs-api-go v1.20.3 github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.6.0 - github.com/prometheus/common v0.10.0 + github.com/prometheus/client_golang v1.8.0 + github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.7.0 - github.com/valyala/fasthttp v1.14.0 + github.com/spf13/viper v1.7.1 + github.com/valyala/fasthttp v1.17.0 go.uber.org/zap v1.16.0 - google.golang.org/grpc v1.33.1 + google.golang.org/grpc v1.33.2 ) replace github.com/nspcc-dev/cdn-neofs-sdk => ../sdk diff --git a/go.sum b/go.sum index 71c40b0..fbf25c9 100644 --- a/go.sum +++ b/go.sum @@ -19,23 +19,35 @@ github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1 github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -51,6 +63,8 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -60,17 +74,24 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -80,15 +101,24 @@ github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KP github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/fasthttp/router v1.1.6 h1:lBcXxp1ZNoNbSeh4+RvAaXKSEiHU6sGd+gEMpd5Xjog= -github.com/fasthttp/router v1.1.6/go.mod h1:E1mpv7mrQzAhiSQdqhRb+GBTC7MEV+bLFVmgzSA5oFM= +github.com/fasthttp/router v1.3.3 h1:CBm3rwvfMhaQa3BCnUO1+XAXNyfTC1KZcSRZaS3Ounk= +github.com/fasthttp/router v1.3.3/go.mod h1:KruUmAmhNcGQtbJcP0len2rDBlnHimjgBpCpGDWAxbE= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -96,15 +126,22 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -117,9 +154,12 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -129,11 +169,14 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -141,12 +184,20 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -157,6 +208,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -167,23 +219,30 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.10.4 h1:jFzIFaf586tquEB5EhzQG0HwGNSlgAJpG53G6Ss11wc= -github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= +github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -191,6 +250,9 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -198,9 +260,11 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -220,6 +284,14 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= @@ -230,8 +302,8 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neofs-api-go v1.20.2 h1:jOsfgOQdKhg7gTAGciRdIdOJxYX72CxqgkwaqzEscHw= -github.com/nspcc-dev/neofs-api-go v1.20.2/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-api-go v1.20.3 h1:V8iqyAaX1WJFiAO79enjETveZfwForBoti6MwdY/Bf8= +github.com/nspcc-dev/neofs-api-go v1.20.3/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= @@ -239,63 +311,98 @@ github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BE github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= -github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= -github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.8.0 h1:zvJNkoCFAnYFNC24FV8nW4JdRJ3GIFcLbg65lL/JDcw= +github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= -github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/savsgio/gotils v0.0.0-20200413113635-8c468ce75cca h1:Qe7Mtuhjkk38HVpRtvWdziZJcwG3Qup1mfyvyOrcnyM= -github.com/savsgio/gotils v0.0.0-20200413113635-8c468ce75cca/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/savsgio/gotils v0.0.0-20200909101946-939aa3fc74fb h1:XPJCVf85HPE2jMVEQ7QWrazaZo1lc94GbUWaQ8Yv5sM= +github.com/savsgio/gotils v0.0.0-20200909101946-939aa3fc74fb/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -303,15 +410,20 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -322,13 +434,15 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.14.0 h1:67bfuW9azCMwW/Jlq/C+VeihNpAuJMWkYPBig1gdi3A= -github.com/valyala/fasthttp v1.14.0/go.mod h1:ol1PCaL0dX20wC0htZ7sYCsvCYmrouYra0zHzaclZhE= +github.com/valyala/fasthttp v1.17.0 h1:P8/koH4aSnJ4xbd0cUUFEGQs3jQqIxoDDyRQrUiAkqg= +github.com/valyala/fasthttp v1.17.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -337,17 +451,25 @@ github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBU go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -359,6 +481,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -389,6 +512,7 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -397,9 +521,12 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -416,6 +543,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -428,23 +556,35 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211 h1:9UQO31fZ+0aKQOFldThf7BKPMJTiBfWycGh/u3UoO88= +golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -464,18 +604,21 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200123022218-593de606220b h1:ztSlcncMErSAUzXwnVO1iTPxHwtvOHBB26SGiyYXIEE= golang.org/x/tools v0.0.0-20200123022218-593de606220b/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -484,46 +627,68 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.33.1 h1:DGeFlSan2f+WEtCERJ4J9GJWk15TxUi8QGagfI87Xyc= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -531,3 +696,5 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= From 31186382e8e9e8fd97d4f8b4ddd04f2daa5a1d5c Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 27 Nov 2020 18:18:53 +0300 Subject: [PATCH 082/548] Change ENV prefix TODO should be replaced with HTTP_GW before release Signed-off-by: Evgeniy Kulikov --- Dockerfile | 7 ++++- main.go | 18 ++++++------ misc.go | 3 ++ settings.go | 84 ++++++++++++++++++++++++++++++++++++++++++++--------- 4 files changed, 88 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index a6f62ba..35365b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,12 @@ COPY . /src ARG VERSION=dev ENV LDFLAGS "-w -s -X main.Version=${VERSION}" RUN set -x \ - && go build -v -mod=vendor -trimpath -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N)" -o /go/bin/neofs-gw ./ \ + && go build \ + -v \ + -mod=vendor \ + -trimpath \ + -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N) -X main.Prefix=HTTP_GW" \ + -o /go/bin/neofs-gw ./ \ && upx -3 /go/bin/neofs-gw # Executable image diff --git a/main.go b/main.go index 36c12f1..823e982 100644 --- a/main.go +++ b/main.go @@ -9,23 +9,23 @@ import ( func newLogger(v *viper.Viper) *zap.Logger { options := []logger.Option{ - logger.WithLevel(v.GetString("logger.level")), - logger.WithTraceLevel(v.GetString("logger.trace_level")), + logger.WithLevel(v.GetString(cfgLoggerLevel)), + logger.WithTraceLevel(v.GetString(cfgLoggerTraceLevel)), - logger.WithFormat(v.GetString("logger.format")), + logger.WithFormat(v.GetString(cfgLoggerFormat)), - logger.WithSamplingInitial(v.GetInt("logger.sampling.initial")), - logger.WithSamplingThereafter(v.GetInt("logger.sampling.thereafter")), + logger.WithSamplingInitial(v.GetInt(cfgLoggerSamplingInitial)), + logger.WithSamplingThereafter(v.GetInt(cfgLoggerSamplingThereafter)), - logger.WithAppName(v.GetString("app_name")), - logger.WithAppVersion(v.GetString("app_version")), + logger.WithAppName(v.GetString(cfgApplicationName)), + logger.WithAppVersion(v.GetString(cfgApplicationVersion)), } - if v.GetBool("logger.no_caller") { + if v.GetBool(cfgLoggerNoCaller) { options = append(options, logger.WithoutCaller()) } - if v.GetBool("logger.no_disclaimer") { + if v.GetBool(cfgLoggerNoDisclaimer) { options = append(options, logger.WithoutDisclaimer()) } diff --git a/misc.go b/misc.go index a2bc3df..ea82ef6 100644 --- a/misc.go +++ b/misc.go @@ -3,4 +3,7 @@ package main var ( Build = "now" Version = "dev" + + // TODO should be replaced with HTTP_GW + Prefix = "GW" ) diff --git a/settings.go b/settings.go index 3a413e4..24bf29e 100644 --- a/settings.go +++ b/settings.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "sort" "strconv" "strings" "time" @@ -28,14 +29,46 @@ const ( defaultKeepaliveTime = 10 * time.Second defaultKeepaliveTimeout = 10 * time.Second + + // Logger: + cfgLoggerLevel = "logger.level" + cfgLoggerFormat = "logger.format" + cfgLoggerTraceLevel = "logger.trace_level" + cfgLoggerNoCaller = "logger.no_caller" + cfgLoggerNoDisclaimer = "logger.no_disclaimer" + cfgLoggerSamplingInitial = "logger.sampling.initial" + cfgLoggerSamplingThereafter = "logger.sampling.thereafter" + + // Peers + cfgPeers = "peers" + + // Application + cfgApplicationName = "app.name" + cfgApplicationVersion = "app.version" + cfgApplicationBuildTime = "app.build_time" + + // command line args + cmdHelp = "help" + cmdVersion = "version" ) +var ignore = map[string]struct{}{ + cfgApplicationName: {}, + cfgApplicationVersion: {}, + cfgApplicationBuildTime: {}, + + cfgPeers: {}, + + cmdHelp: {}, + cmdVersion: {}, +} + func (empty) Read([]byte) (int, error) { return 0, io.EOF } func settings() *viper.Viper { v := viper.New() v.AutomaticEnv() - v.SetEnvPrefix("GW") + v.SetEnvPrefix(Prefix) v.SetConfigType("yaml") v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) @@ -46,8 +79,8 @@ func settings() *viper.Viper { flags.Bool("pprof", false, "enable pprof") flags.Bool("metrics", false, "enable prometheus") - help := flags.BoolP("help", "h", false, "show help") - version := flags.BoolP("version", "v", false, "show version") + help := flags.BoolP(cmdHelp, "h", false, "show help") + version := flags.BoolP(cmdVersion, "v", false, "show version") flags.String("key", generated, `"`+generated+`" to generate key, path to private key file, hex string or wif`) @@ -62,19 +95,19 @@ func settings() *viper.Viper { peers := flags.StringArrayP("peers", "p", nil, "NeoFS nodes") // set prefers: - v.Set("app.name", "neofs-gw") - v.Set("app.version", Version) + v.Set(cfgApplicationName, "neofs-http-gw") + v.Set(cfgApplicationVersion, Version) // set defaults: // logger: - v.SetDefault("logger.level", "debug") - v.SetDefault("logger.format", "console") - v.SetDefault("logger.trace_level", "fatal") - v.SetDefault("logger.no_caller", false) - v.SetDefault("logger.no_disclaimer", true) - v.SetDefault("logger.sampling.initial", 1000) - v.SetDefault("logger.sampling.thereafter", 1000) + v.SetDefault(cfgLoggerLevel, "debug") + v.SetDefault(cfgLoggerFormat, "console") + v.SetDefault(cfgLoggerTraceLevel, "panic") + v.SetDefault(cfgLoggerNoCaller, false) + v.SetDefault(cfgLoggerNoDisclaimer, true) + v.SetDefault(cfgLoggerSamplingInitial, 1000) + v.SetDefault(cfgLoggerSamplingThereafter, 1000) // keepalive: // If set below 10s, a minimum value of 10s will be used instead. @@ -105,6 +138,29 @@ func settings() *viper.Viper { case help != nil && *help: fmt.Printf("NeoFS HTTP Gateway %s (%s)\n", Version, Build) flags.PrintDefaults() + + fmt.Println() + fmt.Println("Default environments:") + fmt.Println() + keys := v.AllKeys() + sort.Strings(keys) + + for i := range keys { + if _, ok := ignore[keys[i]]; ok { + continue + } + + k := strings.Replace(keys[i], ".", "_", -1) + fmt.Printf("%s_%s = %v\n", Prefix, strings.ToUpper(k), v.Get(keys[i])) + } + + fmt.Println() + fmt.Println("Peers preset:") + fmt.Println() + + fmt.Printf("%s_%s_[N]_ADDRESS = string\n", Prefix, strings.ToUpper(cfgPeers)) + fmt.Printf("%s_%s_[N]_WEIGHT = 0..1 (float)\n", Prefix, strings.ToUpper(cfgPeers)) + os.Exit(0) case version != nil && *version: fmt.Printf("NeoFS HTTP Gateway %s (%s)\n", Version, Build) @@ -115,8 +171,8 @@ func settings() *viper.Viper { if peers != nil && len(*peers) > 0 { for i := range *peers { - v.SetDefault("peers."+strconv.Itoa(i)+".address", (*peers)[i]) - v.SetDefault("peers."+strconv.Itoa(i)+".weight", 1) + v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".address", (*peers)[i]) + v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".weight", 1) } } From 01f13ef210d75894e201336ee54802ccfb4d12db Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 2 Dec 2020 11:52:07 +0300 Subject: [PATCH 083/548] Prepare to release - set output for flag set - change env prefix for all environments - migrate to CDN SDK release v0.1.0 Signed-off-by: Evgeniy Kulikov --- Dockerfile | 2 +- go.mod | 4 +--- go.sum | 2 ++ misc.go | 5 ++--- settings.go | 1 + 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 35365b1..ce850dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN set -x \ -v \ -mod=vendor \ -trimpath \ - -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N) -X main.Prefix=HTTP_GW" \ + -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N)" \ -o /go/bin/neofs-gw ./ \ && upx -3 /go/bin/neofs-gw diff --git a/go.mod b/go.mod index c85777c..5c4820b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/fasthttp/router v1.3.3 - github.com/nspcc-dev/cdn-neofs-sdk v0.0.0 + github.com/nspcc-dev/cdn-neofs-sdk v0.1.0 github.com/nspcc-dev/neofs-api-go v1.20.3 github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/pkg/errors v0.9.1 @@ -16,5 +16,3 @@ require ( go.uber.org/zap v1.16.0 google.golang.org/grpc v1.33.2 ) - -replace github.com/nspcc-dev/cdn-neofs-sdk => ../sdk diff --git a/go.sum b/go.sum index fbf25c9..3576898 100644 --- a/go.sum +++ b/go.sum @@ -292,6 +292,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nspcc-dev/cdn-neofs-sdk v0.1.0 h1:nexBQL32WeQC2mvVMiWlOqI2enKcIJlFyxejU1FeBNo= +github.com/nspcc-dev/cdn-neofs-sdk v0.1.0/go.mod h1:ujzmbHoxhuyY4SgHUDSwtfZQNHu06izC4xOkPj+YziY= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= diff --git a/misc.go b/misc.go index ea82ef6..1541334 100644 --- a/misc.go +++ b/misc.go @@ -1,9 +1,8 @@ package main +const Prefix = "HTTP_GW" + var ( Build = "now" Version = "dev" - - // TODO should be replaced with HTTP_GW - Prefix = "GW" ) diff --git a/settings.go b/settings.go index 24bf29e..5b62e6c 100644 --- a/settings.go +++ b/settings.go @@ -74,6 +74,7 @@ func settings() *viper.Viper { // flags setup: flags := pflag.NewFlagSet("commandline", pflag.ExitOnError) + flags.SetOutput(os.Stdout) flags.SortFlags = false flags.Bool("pprof", false, "enable pprof") From 8bd210d67ed65df63fb17acba827839e3920a176 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 2 Dec 2020 12:09:46 +0300 Subject: [PATCH 084/548] Use constant settings keys Signed-off-by: Evgeniy Kulikov --- app.go | 57 +++++++++++++------------------------------ settings.go | 69 ++++++++++++++++++++++++++++++++--------------------- 2 files changed, 59 insertions(+), 67 deletions(-) diff --git a/app.go b/app.go index b40ad53..be68785 100644 --- a/app.go +++ b/app.go @@ -3,8 +3,6 @@ package main import ( "context" "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" "strconv" "github.com/fasthttp/router" @@ -12,7 +10,6 @@ import ( "github.com/nspcc-dev/cdn-neofs-sdk/creds/neofs" "github.com/nspcc-dev/cdn-neofs-sdk/logger" "github.com/nspcc-dev/cdn-neofs-sdk/pool" - crypto "github.com/nspcc-dev/neofs-crypto" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -79,20 +76,20 @@ func newApp(ctx context.Context, opt ...Option) App { a.wlog = logger.GRPC(a.log) - if a.cfg.GetBool("verbose") { + if a.cfg.GetBool(cmdVerbose) { grpclog.SetLoggerV2(a.wlog) } - conTimeout := a.cfg.GetDuration("connect_timeout") - reqTimeout := a.cfg.GetDuration("request_timeout") - tckTimeout := a.cfg.GetDuration("rebalance_timer") + conTimeout := a.cfg.GetDuration(cfgConTimeout) + reqTimeout := a.cfg.GetDuration(cfgReqTimeout) + tckTimeout := a.cfg.GetDuration(cfgRebalance) // -- setup FastHTTP server: -- a.web.Name = "neofs-http-gate" - a.web.ReadBufferSize = a.cfg.GetInt("web.read_buffer_size") - a.web.WriteBufferSize = a.cfg.GetInt("web.write_buffer_size") - a.web.ReadTimeout = a.cfg.GetDuration("web.read_timeout") - a.web.WriteTimeout = a.cfg.GetDuration("web.write_timeout") + a.web.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) + a.web.WriteBufferSize = a.cfg.GetInt(cfgWebWriteBufferSize) + a.web.ReadTimeout = a.cfg.GetDuration(cfgWebReadTimeout) + a.web.WriteTimeout = a.cfg.GetDuration(cfgWebWriteTimeout) a.web.GetOnly = true a.web.DisableHeaderNamesNormalizing = true a.web.NoDefaultServerHeader = true @@ -101,8 +98,8 @@ func newApp(ctx context.Context, opt ...Option) App { connections := make(map[string]float64) for i := 0; ; i++ { - address := a.cfg.GetString("peers." + strconv.Itoa(i) + ".address") - weight := a.cfg.GetFloat64("peers." + strconv.Itoa(i) + ".weight") + address := a.cfg.GetString(cfgPeers + "." + strconv.Itoa(i) + ".address") + weight := a.cfg.GetFloat64(cfgPeers + "." + strconv.Itoa(i) + ".weight") if address == "" { break } @@ -113,7 +110,7 @@ func newApp(ctx context.Context, opt ...Option) App { zap.Float64("weight", weight)) } - cred, err := prepareCredentials(a.cfg.GetString("key"), a.log) + cred, err := neofs.New(a.cfg.GetString(cmdNeoFSKey)) if err != nil { a.log.Fatal("could not prepare credentials", zap.Error(err)) } @@ -130,9 +127,9 @@ func newApp(ctx context.Context, opt ...Option) App { grpc.WithBlock(), grpc.WithInsecure(), grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: a.cfg.GetDuration("keepalive.time"), - Timeout: a.cfg.GetDuration("keepalive.timeout"), - PermitWithoutStream: a.cfg.GetBool("keepalive.permit_without_stream"), + Time: a.cfg.GetDuration(cfgKeepaliveTime), + Timeout: a.cfg.GetDuration(cfgKeepaliveTimeout), + PermitWithoutStream: a.cfg.GetBool(cfgKeepalivePermitWithoutStream), }))) if err != nil { @@ -151,26 +148,6 @@ func newApp(ctx context.Context, opt ...Option) App { return a } -func prepareCredentials(key string, log *zap.Logger) (neofs.Credentials, error) { - if key == generated { - log.Fatal("Don't use generated key, deprecated") - - sk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return nil, err - } - - key, err = crypto.WIFEncode(sk) - if err != nil { - return nil, err - } - - log.Info("generate new key", zap.String("wif", key)) - } - - return neofs.New(key) -} - func (a *app) Wait() { a.log.Info("application started") @@ -204,18 +181,18 @@ func (a *app) Serve(ctx context.Context) { attachHealthy(r, a.pool.Status) // enable metrics - if a.cfg.GetBool("metrics") { + if a.cfg.GetBool(cmdMetrics) { a.log.Info("enabled /metrics/") attachMetrics(r, a.wlog) } // enable pprof - if a.cfg.GetBool("pprof") { + if a.cfg.GetBool(cmdPprof) { a.log.Info("enabled /debug/pprof/") attachProfiler(r) } - bind := a.cfg.GetString("listen_address") + bind := a.cfg.GetString(cfgListenAddress) a.log.Info("run gateway server", zap.String("address", bind)) diff --git a/settings.go b/settings.go index 5b62e6c..a62ccd3 100644 --- a/settings.go +++ b/settings.go @@ -19,10 +19,6 @@ const ( devNull = empty(0) generated = "generated" - minimumTTLInMinutes = 5 - - defaultTTL = minimumTTLInMinutes * time.Minute - defaultRebalanceTimer = 15 * time.Second defaultRequestTimeout = 15 * time.Second defaultConnectTimeout = 30 * time.Second @@ -30,6 +26,25 @@ const ( defaultKeepaliveTime = 10 * time.Second defaultKeepaliveTimeout = 10 * time.Second + cfgListenAddress = "listen_address" + + // KeepAlive + cfgKeepaliveTime = "keepalive.time" + cfgKeepaliveTimeout = "keepalive.timeout" + cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream" + + // Web + cfgWebReadBufferSize = "web.read_buffer_size" + cfgWebWriteBufferSize = "web.write_buffer_size" + cfgWebReadTimeout = "web.read_timeout" + cfgWebWriteTimeout = "web.write_timeout" + cfgWebConnectionPerHost = "web.connection_per_host" + + // Timeouts + cfgConTimeout = "con_timeout" + cfgReqTimeout = "req_timeout" + cfgRebalance = "rebalance_timer" + // Logger: cfgLoggerLevel = "logger.level" cfgLoggerFormat = "logger.format" @@ -48,8 +63,12 @@ const ( cfgApplicationBuildTime = "app.build_time" // command line args - cmdHelp = "help" - cmdVersion = "version" + cmdHelp = "help" + cmdVersion = "version" + cmdVerbose = "verbose" + cmdPprof = "pprof" + cmdMetrics = "metrics" + cmdNeoFSKey = "key" ) var ignore = map[string]struct{}{ @@ -77,23 +96,21 @@ func settings() *viper.Viper { flags.SetOutput(os.Stdout) flags.SortFlags = false - flags.Bool("pprof", false, "enable pprof") - flags.Bool("metrics", false, "enable prometheus") + flags.Bool(cmdPprof, false, "enable pprof") + flags.Bool(cmdMetrics, false, "enable prometheus") help := flags.BoolP(cmdHelp, "h", false, "show help") version := flags.BoolP(cmdVersion, "v", false, "show version") - flags.String("key", generated, `"`+generated+`" to generate key, path to private key file, hex string or wif`) + flags.String(cmdNeoFSKey, "", `"Path to private key file, hex string or wif`) - flags.Bool("verbose", false, "debug gRPC connections") - flags.Duration("request_timeout", defaultRequestTimeout, "gRPC request timeout") - flags.Duration("connect_timeout", defaultConnectTimeout, "gRPC connect timeout") - flags.Duration("rebalance_timer", defaultRebalanceTimer, "gRPC connection rebalance timer") + flags.Bool(cmdVerbose, false, "debug gRPC connections") + flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout") + flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout") + flags.Duration(cfgRebalance, defaultRebalanceTimer, "gRPC connection rebalance timer") - ttl := flags.DurationP("conn_ttl", "t", defaultTTL, "gRPC connection time to live") - - flags.String("listen_address", "0.0.0.0:8082", "HTTP Gateway listen address") - peers := flags.StringArrayP("peers", "p", nil, "NeoFS nodes") + flags.String(cfgListenAddress, "0.0.0.0:8082", "HTTP Gateway listen address") + peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") // set prefers: v.Set(cfgApplicationName, "neofs-http-gw") @@ -112,16 +129,16 @@ func settings() *viper.Viper { // keepalive: // If set below 10s, a minimum value of 10s will be used instead. - v.SetDefault("keepalive.time", defaultKeepaliveTime) - v.SetDefault("keepalive.timeout", defaultKeepaliveTimeout) - v.SetDefault("keepalive.permit_without_stream", true) + v.SetDefault(cfgKeepaliveTime, defaultKeepaliveTime) + v.SetDefault(cfgKeepaliveTimeout, defaultKeepaliveTimeout) + v.SetDefault(cfgKeepalivePermitWithoutStream, true) // web-server: - v.SetDefault("web.read_buffer_size", 4096) - v.SetDefault("web.write_buffer_size", 4096) - v.SetDefault("web.read_timeout", time.Second*15) - v.SetDefault("web.write_timeout", time.Minute) - v.SetDefault("web.connection_per_host", 10) + v.SetDefault(cfgWebReadBufferSize, 4096) + v.SetDefault(cfgWebWriteBufferSize, 4096) + v.SetDefault(cfgWebReadTimeout, time.Second*15) + v.SetDefault(cfgWebWriteTimeout, time.Minute) + v.SetDefault(cfgWebConnectionPerHost, 10) if err := v.BindPFlags(flags); err != nil { panic(err) @@ -166,8 +183,6 @@ func settings() *viper.Viper { case version != nil && *version: fmt.Printf("NeoFS HTTP Gateway %s (%s)\n", Version, Build) os.Exit(0) - case ttl != nil && ttl.Minutes() < minimumTTLInMinutes: - fmt.Printf("connection ttl should not be less than %s", defaultTTL) } if peers != nil && len(*peers) > 0 { From 65a5a3f9fc66956a24e44f1a3807dc839c4adac6 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 2 Dec 2020 12:20:21 +0300 Subject: [PATCH 085/548] Update README Signed-off-by: Evgeniy Kulikov --- README.md | 38 ++++++++++++++++++++++++-------------- settings.go | 4 ++-- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 7b63d14..0a9831b 100644 --- a/README.md +++ b/README.md @@ -24,21 +24,31 @@ You can download files from NeoFS Network using NeoFS Gate. # Environments: -GW_KEY=stirng - "generated" to generate key, path to private key file, hex string or wif (default "generated") -GW_REQUEST_TIMEOUT=Duration - timeout for request -GW_CONNECT_TIMEOUT=Duration - timeout for connection -GW_LISTEN_ADDRESS=host:port - address to listen connections -GW_PEERS__ADDRESS=host:port - address of NeoFS Node -GW_PEERS__WEIGHT=float - weight of NeoFS Node -GW_PPROF=bool - enable/disable pprof (/debug/pprof) -GW_METRICS=bool - enable/disable prometheus metrics endpoint (/metrics) -GW_KEEPALIVE_TIME=Duration - аfter a duration of this time if the client doesn't see any activity +HTTP_GW_KEY=string - "generated" to generate key, path to private key file, hex string or wif (default "generated") +HTTP_GW_CONNECT_TIMEOUT=Duration - timeout for connection +HTTP_GW_REQUEST_TIMEOUT=Duration - timeout for request +HTTP_GW_REBALANCE_TIMER=Duration - time between connections checks +HTTP_GW_LISTEN_ADDRESS=host:port - address to listen connections +HTTP_GW_PEERS__ADDRESS=host:port - address of NeoFS Node +HTTP_GW_PEERS__WEIGHT=float - weight of NeoFS Node +HTTP_GW_PPROF=bool - enable/disable pprof (/debug/pprof) +HTTP_GW_METRICS=bool - enable/disable prometheus metrics endpoint (/metrics) +HTTP_GW_LOGGER_FORMAT=string - logger format +HTTP_GW_LOGGER_LEVEL=string - logger level +HTTP_GW_LOGGER_NO_CALLER=bool - logger don't show caller +HTTP_GW_LOGGER_NO_DISCLAIMER=bool - logger don't show application name/version +HTTP_GW_LOGGER_SAMPLING_INITIAL=int - logger sampling initial +HTTP_GW_LOGGER_SAMPLING_THEREAFTER=int - logger sampling thereafter +HTTP_GW_LOGGER_TRACE_LEVEL=string - logger show trace on level +HTTP_GW_KEEPALIVE_TIME=Duration - аfter a duration of this time if the client doesn't see any activity it pings the server to see if the transport is still alive. -GW_KEEPALIVE_TIMEOUT=Duration - after having pinged for keepalive check, the client waits for a duration +HTTP_GW_KEEPALIVE_TIMEOUT=Duration - after having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed -GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=Bool - if true, client sends keepalive pings even with no active RPCs. +HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=Bool - if true, client sends keepalive pings even with no active RPCs. If false, when there are no active RPCs, Time and Timeout will be ignored and no keepalive pings will be sent. -``` -### WARNING -`generated` value for `GW_KEY` or `--key` is deprecated, you should use pre-generated keys. \ No newline at end of file +Peers preset: + +HTTP_GW_PEERS_[N]_ADDRESS = string +HTTP_GW_PEERS_[N]_WEIGHT = 0..1 (float) +``` \ No newline at end of file diff --git a/settings.go b/settings.go index a62ccd3..c781056 100644 --- a/settings.go +++ b/settings.go @@ -41,8 +41,8 @@ const ( cfgWebConnectionPerHost = "web.connection_per_host" // Timeouts - cfgConTimeout = "con_timeout" - cfgReqTimeout = "req_timeout" + cfgConTimeout = "connect_timeout" + cfgReqTimeout = "request_timeout" cfgRebalance = "rebalance_timer" // Logger: From a18e4200b05bcb393d131202b18d19ba3c76a65a Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 2 Dec 2020 12:22:27 +0300 Subject: [PATCH 086/548] Cleanup dependencies Signed-off-by: Evgeniy Kulikov --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 5c4820b..f7387e3 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/fasthttp/router v1.3.3 github.com/nspcc-dev/cdn-neofs-sdk v0.1.0 github.com/nspcc-dev/neofs-api-go v1.20.3 - github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.8.0 github.com/prometheus/common v0.15.0 From 77c404d78fc952f14b456ba4b6db55c76b328b22 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 3 Dec 2020 18:46:36 +0300 Subject: [PATCH 087/548] Add how to check HTTP Gate --- docs/how-to-check.mc | 111 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 docs/how-to-check.mc diff --git a/docs/how-to-check.mc b/docs/how-to-check.mc new file mode 100644 index 0000000..ca96116 --- /dev/null +++ b/docs/how-to-check.mc @@ -0,0 +1,111 @@ +# How to check + +1. Create container +``` +→ neofs-cli -k /path/to/user.key -r s01.neofs.devenv:8080 container create --name TestStorage --basic-acl public -p "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" --await +container ID: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM +awaiting... +container has been persisted on sidechain +``` + +2. Put object into container + +``` +→ neofs-cli -k /path/to/user.key -r s01.neofs.devenv:8080 object put --cid 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM --file /path/to/1.jpeg +[/path/to/1.jpeg] Object successfully stored + ID: GTUokhLMtEq1Kh1nzSVCsWybFvVHFQyhZHaVXZBrYd3W + CID: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM +``` + +3. Check that object can be fetched by oldest API + +``` +→ curl -sSI -XGET http://http.neofs.devenv/get/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/GTUokhLMtEq1Kh1nzSVCsWybFvVHFQyhZHaVXZBrYd3W +HTTP/1.1 200 OK +Date: Thu, 03 Dec 2020 15:04:52 GMT +Content-Type: image/jpeg +Content-Length: 93077 +x-object-id: GTUokhLMtEq1Kh1nzSVCsWybFvVHFQyhZHaVXZBrYd3W +x-owner-id: NTrezR3C4X8aMLVg7vozt5wguyNfFhwuFx +x-container-id: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM +x-FileName: 1.jpeg +x-Timestamp: 1607006318 +Last-Modified: Thu, 03 Dec 2020 17:38:38 MSK +Content-Disposition: inline; filename=1.jpeg +``` + +4. Check that object can be fetched by newest API + +``` +→ curl -sSI -XGET http://http.neofs.devenv/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/FileName/1.jpeg +HTTP/1.1 200 OK +Date: Thu, 03 Dec 2020 15:04:52 GMT +Content-Type: image/jpeg +Content-Length: 93077 +x-object-id: GTUokhLMtEq1Kh1nzSVCsWybFvVHFQyhZHaVXZBrYd3W +x-owner-id: NTrezR3C4X8aMLVg7vozt5wguyNfFhwuFx +x-container-id: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM +x-FileName: 1.jpeg +x-Timestamp: 1607006318 +Last-Modified: Thu, 03 Dec 2020 17:38:38 MSK +Content-Disposition: inline; filename=1.jpeg +``` + +5. Put second object with same name + +``` +→ neofs-cli -k /path/to/user.key -r s01.neofs.devenv:8080 object put --cid 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM --file /path/to/1.jpeg +[/path/to/1.jpeg] Object successfully stored + ID: 14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga + CID: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM + +``` + +6. Check that object can be fetched by oldest API + +``` +→ curl -sSI -XGET http://http.neofs.devenv/get/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga +HTTP/1.1 200 OK +Date: Thu, 03 Dec 2020 15:07:51 GMT +Content-Type: image/jpeg +Content-Length: 93077 +x-object-id: 14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga +x-owner-id: NTrezR3C4X8aMLVg7vozt5wguyNfFhwuFx +x-container-id: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM +x-FileName: 1.jpeg +x-Timestamp: 1607006355 +Last-Modified: Thu, 03 Dec 2020 17:39:15 MSK +Content-Disposition: inline; filename=1.jpeg +``` + +7. Retry fetch object by newest API + +``` +→ curl -sSI -XGET http://http.neofs.devenv/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/FileName/1.jpeg +HTTP/1.1 200 OK +Date: Thu, 03 Dec 2020 15:04:28 GMT +Content-Type: image/jpeg +Content-Length: 93077 +x-object-id: 14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga +x-owner-id: NTrezR3C4X8aMLVg7vozt5wguyNfFhwuFx +x-container-id: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM +x-FileName: 1.jpeg +x-Timestamp: 1607006355 +Last-Modified: Thu, 03 Dec 2020 17:39:15 MSK +Content-Disposition: inline; filename=1.jpeg +``` + +**http-gate log when find multiple objects** +``` +2020-12-03T18:04:28.617+0300 debug neofs-gw/receive.go:191 find multiple objects {"cid": "88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM", "attr_key": "FileName", "attr_val": "1.jpeg", "object_ids": ["14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga", "GTUokhLMtEq1Kh1nzSVCsWybFvVHFQyhZHaVXZBrYd3W"], "show_object_id": "14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga"} +``` + +8. Check newest API when object not found + +``` +→ curl -sSI -XGET http://http.neofs.devenv/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/FileName/2.jpeg +HTTP/1.1 404 Not Found +Date: Thu, 03 Dec 2020 15:11:07 GMT +Content-Type: text/plain; charset=utf-8 +Content-Length: 9 +``` From abd95efe4749e0b1986f46ea0070d627f1268ec5 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 3 Dec 2020 18:00:43 +0300 Subject: [PATCH 088/548] Refactoring HTTP Gate - simplify code - add `/get_by_attribute` - refactoring receiving object Signed-off-by: Evgeniy Kulikov --- app.go | 4 +- receive.go | 155 +++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 118 insertions(+), 41 deletions(-) diff --git a/app.go b/app.go index be68785..e0ffd63 100644 --- a/app.go +++ b/app.go @@ -175,7 +175,9 @@ func (a *app) Serve(ctx context.Context) { r.RedirectTrailingSlash = true a.log.Info("enabled /get/{cid}/{oid}") - r.GET("/get/{cid}/{oid}", a.receiveFile) + r.GET("/get/{cid}/{oid}", a.byAddress) + a.log.Info("enabled /get_by_attribute/{cid}/{attr_key}/{attr_val}") + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val}", a.byAttribute) // attaching /-/(ready,healthy) attachHealthy(r, a.pool.Status) diff --git a/receive.go b/receive.go index 5688b0f..2af422b 100644 --- a/receive.go +++ b/receive.go @@ -10,6 +10,7 @@ import ( "time" sdk "github.com/nspcc-dev/cdn-neofs-sdk" + "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/pkg/errors" "github.com/valyala/fasthttp" @@ -18,12 +19,23 @@ import ( "google.golang.org/grpc/status" ) -type detector struct { - io.Writer - sync.Once +type ( + detector struct { + io.Writer + sync.Once - contentType string -} + contentType string + } + + request struct { + *fasthttp.RequestCtx + + log *zap.Logger + obj sdk.ObjectClient + } + + objectIDs []*object.ID +) func newDetector(w io.Writer) *detector { return &detector{Writer: w} @@ -37,34 +49,18 @@ func (d *detector) Write(data []byte) (int, error) { return d.Writer.Write(data) } -func (a *app) receiveFile(c *fasthttp.RequestCtx) { +func (r *request) receiveFile(address *object.Address) { var ( - err error - disp = "inline" - start = time.Now() - address = object.NewAddress() - sCID, _ = c.UserValue("cid").(string) - sOID, _ = c.UserValue("oid").(string) - value = strings.Join([]string{sCID, sOID}, "/") - + err error + dis = "inline" + start = time.Now() filename string ) - log := a.log.With( - // zap.String("node", con.Target()), - zap.String("cid", sCID), - zap.String("oid", sOID)) - - if err = address.Parse(value); err != nil { - log.Error("wrong object address", zap.Error(err)) - c.Error("wrong object address", fasthttp.StatusBadRequest) - return - } - - writer := newDetector(c.Response.BodyWriter()) - obj, err := a.cli.Object().Get(c, address, sdk.WithGetWriter(writer)) + writer := newDetector(r.Response.BodyWriter()) + obj, err := r.obj.Get(r, address, sdk.WithGetWriter(writer)) if err != nil { - log.Error("could not receive object", + r.log.Error("could not receive object", zap.Stringer("elapsed", time.Since(start)), zap.Error(err)) @@ -81,24 +77,24 @@ func (a *app) receiveFile(c *fasthttp.RequestCtx) { msg = st.Message() } - c.Error(msg, code) + r.Error(msg, code) return } - if c.Request.URI().QueryArgs().GetBool("download") { - disp = "attachment" + if r.Request.URI().QueryArgs().GetBool("download") { + dis = "attachment" } - c.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) - c.Response.Header.Set("x-object-id", obj.ID().String()) - c.Response.Header.Set("x-owner-id", obj.OwnerID().String()) - c.Response.Header.Set("x-container-id", obj.ContainerID().String()) + r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) + r.Response.Header.Set("x-object-id", obj.ID().String()) + r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) + r.Response.Header.Set("x-container-id", obj.ContainerID().String()) for _, attr := range obj.Attributes() { key := attr.Key() val := attr.Value() - c.Response.Header.Set("x-"+key, val) + r.Response.Header.Set("x-"+key, val) switch key { case object.AttributeFileName: @@ -106,19 +102,98 @@ func (a *app) receiveFile(c *fasthttp.RequestCtx) { case object.AttributeTimestamp: value, err := strconv.ParseInt(val, 10, 64) if err != nil { - a.log.Info("couldn't parse creation date", + r.log.Info("couldn't parse creation date", zap.String("key", key), zap.String("val", val), zap.Error(err)) continue } - c.Response.Header.Set("Last-Modified", + r.Response.Header.Set("Last-Modified", time.Unix(value, 0).Format(time.RFC1123)) } } - c.SetContentType(writer.contentType) - c.Response.Header.Set("Content-Disposition", disp+"; filename="+path.Base(filename)) + r.SetContentType(writer.contentType) + r.Response.Header.Set("Content-Disposition", dis+"; filename="+path.Base(filename)) +} + +func (o objectIDs) Slice() []string { + res := make([]string, 0, len(o)) + for _, oid := range o { + res = append(res, oid.String()) + } + + return res +} + +func (a *app) request(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { + return &request{ + RequestCtx: ctx, + + log: log, + obj: a.cli.Object(), + } +} + +func (a *app) byAddress(c *fasthttp.RequestCtx) { + var ( + err error + adr = object.NewAddress() + cid, _ = c.UserValue("cid").(string) + oid, _ = c.UserValue("oid").(string) + val = strings.Join([]string{cid, oid}, "/") + log = a.log.With( + zap.String("cid", cid), + zap.String("oid", oid)) + ) + + if err = adr.Parse(val); err != nil { + log.Error("wrong object address", zap.Error(err)) + c.Error("wrong object address", fasthttp.StatusBadRequest) + return + } + + a.request(c, log).receiveFile(adr) +} + +func (a *app) byAttribute(c *fasthttp.RequestCtx) { + var ( + err error + ids []*object.ID + cid = container.NewID() + adr = object.NewAddress() + sCID, _ = c.UserValue("cid").(string) + key, _ = c.UserValue("attr_key").(string) + val, _ = c.UserValue("attr_val").(string) + + log = a.log.With( + zap.String("cid", sCID), + zap.String("attr_key", key), + zap.String("attr_val", val)) + ) + + if err = cid.Parse(sCID); err != nil { + log.Error("wrong container id", zap.Error(err)) + c.Error("wrong container id", fasthttp.StatusBadRequest) + return + } else if ids, err = a.cli.Object().Search(c, cid, sdk.SearchRootObjects(), sdk.SearchByAttribute(key, val)); err != nil { + log.Error("something went wrong", zap.Error(err)) + c.Error("something went wrong", fasthttp.StatusBadRequest) + return + } else if len(ids) == 0 { + c.Error("not found", fasthttp.StatusNotFound) + return + } + + if len(ids) > 1 { + log.Debug("found multiple objects", + zap.Strings("object_ids", objectIDs(ids).Slice()), + zap.Stringer("show_object_id", ids[0])) + } + + adr.SetContainerID(cid) + adr.SetObjectID(ids[0]) + a.request(c, log).receiveFile(adr) } From 4fb7e511dc87e646b25a92b8e48c9ef55475e657 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 3 Dec 2020 19:12:00 +0300 Subject: [PATCH 089/548] Rename how-to-check.mc to how-to-check.md --- docs/{how-to-check.mc => how-to-check.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{how-to-check.mc => how-to-check.md} (100%) diff --git a/docs/how-to-check.mc b/docs/how-to-check.md similarity index 100% rename from docs/how-to-check.mc rename to docs/how-to-check.md From ac8affff91e05a7140b1acee2164b792a663162c Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 23 Dec 2020 14:21:34 +0300 Subject: [PATCH 090/548] Migrate to new release SDK --- go.mod | 6 +++--- go.sum | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index f7387e3..7017089 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.13 require ( github.com/fasthttp/router v1.3.3 - github.com/nspcc-dev/cdn-neofs-sdk v0.1.0 - github.com/nspcc-dev/neofs-api-go v1.20.3 + github.com/nspcc-dev/cdn-neofs-sdk v0.2.0 + github.com/nspcc-dev/neofs-api-go v1.21.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.8.0 github.com/prometheus/common v0.15.0 @@ -13,5 +13,5 @@ require ( github.com/spf13/viper v1.7.1 github.com/valyala/fasthttp v1.17.0 go.uber.org/zap v1.16.0 - google.golang.org/grpc v1.33.2 + google.golang.org/grpc v1.34.0 ) diff --git a/go.sum b/go.sum index 3576898..a1b3754 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -111,6 +112,7 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp/router v1.3.3 h1:CBm3rwvfMhaQa3BCnUO1+XAXNyfTC1KZcSRZaS3Ounk= @@ -292,8 +294,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nspcc-dev/cdn-neofs-sdk v0.1.0 h1:nexBQL32WeQC2mvVMiWlOqI2enKcIJlFyxejU1FeBNo= -github.com/nspcc-dev/cdn-neofs-sdk v0.1.0/go.mod h1:ujzmbHoxhuyY4SgHUDSwtfZQNHu06izC4xOkPj+YziY= +github.com/nspcc-dev/cdn-neofs-sdk v0.2.0 h1:y3pwoFKsHSJZYgW7Nty9xiktLgXfCv9I7+hPN9Q9O3s= +github.com/nspcc-dev/cdn-neofs-sdk v0.2.0/go.mod h1:3dPsQIGuVuiwMbVWFCOKyBQvHbcg++AA+st1Pkwd+ms= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= @@ -304,8 +306,8 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neofs-api-go v1.20.3 h1:V8iqyAaX1WJFiAO79enjETveZfwForBoti6MwdY/Bf8= -github.com/nspcc-dev/neofs-api-go v1.20.3/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-api-go v1.21.0 h1:vQsiWTYWKglswdx1oAoriJDn9cU9wA528uqnNcV2nbA= +github.com/nspcc-dev/neofs-api-go v1.21.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= @@ -431,6 +433,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= @@ -464,6 +467,8 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= @@ -650,10 +655,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.33.1 h1:DGeFlSan2f+WEtCERJ4J9GJWk15TxUi8QGagfI87Xyc= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From c30b264beb4041de83e4f28d446ef19d0d1ac3bc Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 14 Jan 2021 13:28:20 +0300 Subject: [PATCH 091/548] Migrate to SDK 0.3.0 - update dependencies - github.com/fasthttp/router v1.3.5 - github.com/nspcc-dev/cdn-sdk v0.3.0 - github.com/nspcc-dev/neofs-api-go v1.22.0 - github.com/prometheus/client_golang v1.9.0 - github.com/valyala/fasthttp v1.19.0 - google.golang.org/grpc v1.35.0 Signed-off-by: Evgeniy Kulikov --- app.go | 8 ++++---- go.mod | 12 ++++++------ go.sum | 30 ++++++++++++++++-------------- main.go | 4 ++-- receive.go | 2 +- 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/app.go b/app.go index e0ffd63..17909e7 100644 --- a/app.go +++ b/app.go @@ -6,10 +6,10 @@ import ( "strconv" "github.com/fasthttp/router" - sdk "github.com/nspcc-dev/cdn-neofs-sdk" - "github.com/nspcc-dev/cdn-neofs-sdk/creds/neofs" - "github.com/nspcc-dev/cdn-neofs-sdk/logger" - "github.com/nspcc-dev/cdn-neofs-sdk/pool" + sdk "github.com/nspcc-dev/cdn-sdk" + "github.com/nspcc-dev/cdn-sdk/creds/neofs" + "github.com/nspcc-dev/cdn-sdk/logger" + "github.com/nspcc-dev/cdn-sdk/pool" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" diff --git a/go.mod b/go.mod index 7017089..5cbf74c 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,17 @@ module github.com/nspcc-dev/neofs-http-gate -go 1.13 +go 1.15 require ( - github.com/fasthttp/router v1.3.3 - github.com/nspcc-dev/cdn-neofs-sdk v0.2.0 - github.com/nspcc-dev/neofs-api-go v1.21.0 + github.com/fasthttp/router v1.3.5 + github.com/nspcc-dev/cdn-sdk v0.3.0 + github.com/nspcc-dev/neofs-api-go v1.22.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.8.0 github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 - github.com/valyala/fasthttp v1.17.0 + github.com/valyala/fasthttp v1.19.0 go.uber.org/zap v1.16.0 - google.golang.org/grpc v1.34.0 + google.golang.org/grpc v1.35.0 ) diff --git a/go.sum b/go.sum index a1b3754..2faf92b 100644 --- a/go.sum +++ b/go.sum @@ -77,7 +77,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -112,11 +112,11 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/fasthttp/router v1.3.3 h1:CBm3rwvfMhaQa3BCnUO1+XAXNyfTC1KZcSRZaS3Ounk= -github.com/fasthttp/router v1.3.3/go.mod h1:KruUmAmhNcGQtbJcP0len2rDBlnHimjgBpCpGDWAxbE= +github.com/fasthttp/router v1.3.5 h1:XGZZtcYUH+9ZoLxKhwF66F8sGnMT+Jg00j0JdVzwmDA= +github.com/fasthttp/router v1.3.5/go.mod h1:BylQKgvh6YQkR0mvL60+HJyTaGwcn5d8UFNweOb/Nw8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -294,8 +294,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nspcc-dev/cdn-neofs-sdk v0.2.0 h1:y3pwoFKsHSJZYgW7Nty9xiktLgXfCv9I7+hPN9Q9O3s= -github.com/nspcc-dev/cdn-neofs-sdk v0.2.0/go.mod h1:3dPsQIGuVuiwMbVWFCOKyBQvHbcg++AA+st1Pkwd+ms= +github.com/nspcc-dev/cdn-sdk v0.3.0 h1:9Q46p/I299ZX/kvpfWvForouRrEhWWVlAJwwMnzsKzc= +github.com/nspcc-dev/cdn-sdk v0.3.0/go.mod h1:NsfdU96++IfZIqSjhwCyGjaT+1tp09C9VKE9QOtjeuQ= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= @@ -306,8 +306,8 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neofs-api-go v1.21.0 h1:vQsiWTYWKglswdx1oAoriJDn9cU9wA528uqnNcV2nbA= -github.com/nspcc-dev/neofs-api-go v1.21.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-api-go v1.22.0 h1:tMFlxDTzoNnSMMXHUHTCJ9DGPO2FUkOQxb/iSuk1kiU= +github.com/nspcc-dev/neofs-api-go v1.22.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= @@ -394,8 +394,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/savsgio/gotils v0.0.0-20200909101946-939aa3fc74fb h1:XPJCVf85HPE2jMVEQ7QWrazaZo1lc94GbUWaQ8Yv5sM= -github.com/savsgio/gotils v0.0.0-20200909101946-939aa3fc74fb/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= +github.com/savsgio/gotils v0.0.0-20210105085219-0567298fdcac h1:5pr1F6tcjeIKPKlrQZW6hzB0WxZge7SkkYVGG/e5pLY= +github.com/savsgio/gotils v0.0.0-20210105085219-0567298fdcac/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -436,6 +436,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= @@ -446,8 +448,8 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.17.0 h1:P8/koH4aSnJ4xbd0cUUFEGQs3jQqIxoDDyRQrUiAkqg= -github.com/valyala/fasthttp v1.17.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= +github.com/valyala/fasthttp v1.19.0 h1:PfTS4PeH3xDr3WomrDS2ID8lU2GskK1xS3YG6gIpibU= +github.com/valyala/fasthttp v1.19.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -655,8 +657,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/main.go b/main.go index 823e982..770d5a8 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,8 @@ package main import ( - "github.com/nspcc-dev/cdn-neofs-sdk/grace" - "github.com/nspcc-dev/cdn-neofs-sdk/logger" + "github.com/nspcc-dev/cdn-sdk/grace" + "github.com/nspcc-dev/cdn-sdk/logger" "github.com/spf13/viper" "go.uber.org/zap" ) diff --git a/receive.go b/receive.go index 2af422b..a290b79 100644 --- a/receive.go +++ b/receive.go @@ -9,7 +9,7 @@ import ( "sync" "time" - sdk "github.com/nspcc-dev/cdn-neofs-sdk" + sdk "github.com/nspcc-dev/cdn-sdk" "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/pkg/errors" From 52812f78690e023cc0ad4c338a6314927ab99d15 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 18 Jan 2021 10:08:44 +0300 Subject: [PATCH 092/548] Release v0.14.1 with CDN SDK v0.3.1 and NeoFS API Go v1.22.1 --- go.mod | 6 +++--- go.sum | 17 ++++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 5cbf74c..33834cc 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.15 require ( github.com/fasthttp/router v1.3.5 - github.com/nspcc-dev/cdn-sdk v0.3.0 - github.com/nspcc-dev/neofs-api-go v1.22.0 + github.com/nspcc-dev/cdn-sdk v0.3.1 + github.com/nspcc-dev/neofs-api-go v1.22.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.8.0 + github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 diff --git a/go.sum b/go.sum index 2faf92b..8ea2a6a 100644 --- a/go.sum +++ b/go.sum @@ -294,8 +294,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nspcc-dev/cdn-sdk v0.3.0 h1:9Q46p/I299ZX/kvpfWvForouRrEhWWVlAJwwMnzsKzc= -github.com/nspcc-dev/cdn-sdk v0.3.0/go.mod h1:NsfdU96++IfZIqSjhwCyGjaT+1tp09C9VKE9QOtjeuQ= +github.com/nspcc-dev/cdn-sdk v0.3.1 h1:8aDktq8vkzv/KUF1isSe6sUQiDGn6CG1k0Y/zaNn1LA= +github.com/nspcc-dev/cdn-sdk v0.3.1/go.mod h1:Wiw3oQjT2F+8ZRt8VnC34ZrSuCDvijyxc1p7bWmcSvk= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= @@ -306,8 +306,8 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neofs-api-go v1.22.0 h1:tMFlxDTzoNnSMMXHUHTCJ9DGPO2FUkOQxb/iSuk1kiU= -github.com/nspcc-dev/neofs-api-go v1.22.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-api-go v1.22.1 h1:O4uRUM4OxAKKQOebqXIn8uDaER4E5LTEO9Yd7GYkiQA= +github.com/nspcc-dev/neofs-api-go v1.22.1/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= @@ -358,8 +358,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.8.0 h1:zvJNkoCFAnYFNC24FV8nW4JdRJ3GIFcLbg65lL/JDcw= -github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM= +github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU= +github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -374,7 +374,6 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -579,8 +578,8 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211 h1:9UQO31fZ+0aKQOFldThf7BKPMJTiBfWycGh/u3UoO88= -golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= From b6d94ec9a528c75d44384e03fb52747eee81807e Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Sat, 23 Jan 2021 16:28:12 +0300 Subject: [PATCH 093/548] Fixes around `get_by_attribute` Before, `attr_val` ignores attribute values that contains slashes in it value, now it's fixed. Update to v0.3.2 with fixing status of connection pool. Added debug log message when object not found, for debug reasons. Update .test.env variables prefix, it was changed in one of previous releases. Signed-off-by: Evgeniy Kulikov --- .test.env | 36 ++++++++++++++++++------------------ app.go | 5 +++-- go.mod | 2 +- go.sum | 4 ++-- receive.go | 1 + 5 files changed, 25 insertions(+), 23 deletions(-) diff --git a/.test.env b/.test.env index c3ac45d..569bfda 100644 --- a/.test.env +++ b/.test.env @@ -1,19 +1,19 @@ -GW_VERBOSE=--true -GW_KEY=generated +HTTP_GW_VERBOSE=--true +HTTP_GW_KEY=generated -GW_LISTEN_ADDRESS=0.0.0.0:8087 -GW_LOGGER_LEVEL=debug -GW_CONNECT_TIMEOUT=60s -GW_REQUEST_TIMEOUT=300s -GW_KEEPALIVE_TIMEOUT=300s -GW_KEEPALIVE_TIME=120s -GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=True -GW_CONN_TTL=1h -GW_PEERS_0_WEIGHT=1.0 -GW_PEERS_1_WEIGHT=0.125 -GW_PEERS_2_WEIGHT=0.125 -GW_PEERS_3_WEIGHT=0.125 -GW_PEERS_4_WEIGHT=0.125 -GW_PEERS_5_WEIGHT=0.125 -GW_PEERS_6_WEIGHT=0.125 -GW_PEERS_7_WEIGHT=0.125 \ No newline at end of file +HTTP_GW_LISTEN_ADDRESS=0.0.0.0:8087 +HTTP_GW_LOGGER_LEVEL=debug +HTTP_GW_CONNECT_TIMEOUT=60s +HTTP_GW_REQUEST_TIMEOUT=300s +HTTP_GW_KEEPALIVE_TIMEOUT=300s +HTTP_GW_KEEPALIVE_TIME=120s +HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=True +HTTP_GW_CONN_TTL=1h +HTTP_GW_PEERS_0_WEIGHT=1.0 +HTTP_GW_PEERS_1_WEIGHT=0.125 +HTTP_GW_PEERS_2_WEIGHT=0.125 +HTTP_GW_PEERS_3_WEIGHT=0.125 +HTTP_GW_PEERS_4_WEIGHT=0.125 +HTTP_GW_PEERS_5_WEIGHT=0.125 +HTTP_GW_PEERS_6_WEIGHT=0.125 +HTTP_GW_PEERS_7_WEIGHT=0.125 \ No newline at end of file diff --git a/app.go b/app.go index 17909e7..4b56ac3 100644 --- a/app.go +++ b/app.go @@ -176,8 +176,9 @@ func (a *app) Serve(ctx context.Context) { a.log.Info("enabled /get/{cid}/{oid}") r.GET("/get/{cid}/{oid}", a.byAddress) - a.log.Info("enabled /get_by_attribute/{cid}/{attr_key}/{attr_val}") - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val}", a.byAttribute) + + a.log.Info("enabled /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.byAttribute) // attaching /-/(ready,healthy) attachHealthy(r, a.pool.Status) diff --git a/go.mod b/go.mod index 33834cc..0b1e740 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.15 require ( github.com/fasthttp/router v1.3.5 - github.com/nspcc-dev/cdn-sdk v0.3.1 + github.com/nspcc-dev/cdn-sdk v0.3.2 github.com/nspcc-dev/neofs-api-go v1.22.1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.9.0 diff --git a/go.sum b/go.sum index 8ea2a6a..125b0ce 100644 --- a/go.sum +++ b/go.sum @@ -294,8 +294,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nspcc-dev/cdn-sdk v0.3.1 h1:8aDktq8vkzv/KUF1isSe6sUQiDGn6CG1k0Y/zaNn1LA= -github.com/nspcc-dev/cdn-sdk v0.3.1/go.mod h1:Wiw3oQjT2F+8ZRt8VnC34ZrSuCDvijyxc1p7bWmcSvk= +github.com/nspcc-dev/cdn-sdk v0.3.2 h1:R3OCj9yF1PKxPr/HQWTriNOHfsIq/wvSeGYnYkDtVAo= +github.com/nspcc-dev/cdn-sdk v0.3.2/go.mod h1:Wiw3oQjT2F+8ZRt8VnC34ZrSuCDvijyxc1p7bWmcSvk= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= diff --git a/receive.go b/receive.go index a290b79..5139252 100644 --- a/receive.go +++ b/receive.go @@ -183,6 +183,7 @@ func (a *app) byAttribute(c *fasthttp.RequestCtx) { c.Error("something went wrong", fasthttp.StatusBadRequest) return } else if len(ids) == 0 { + log.Debug("object not found") c.Error("not found", fasthttp.StatusNotFound) return } From c2c225988da55f5a6e3e0f1d0bf94268375dc5c1 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 1 Feb 2021 17:31:50 +0300 Subject: [PATCH 094/548] Update NeoFS API an SDK for HTTP GW - github.com/nspcc-dev/cdn-sdk v0.3.3 - github.com/nspcc-dev/neofs-api-go v1.22.2 Signed-off-by: Evgeniy Kulikov --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0b1e740..dbfbb98 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.15 require ( github.com/fasthttp/router v1.3.5 - github.com/nspcc-dev/cdn-sdk v0.3.2 - github.com/nspcc-dev/neofs-api-go v1.22.1 + github.com/nspcc-dev/cdn-sdk v0.3.3 + github.com/nspcc-dev/neofs-api-go v1.22.2 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 diff --git a/go.sum b/go.sum index 125b0ce..2bb3103 100644 --- a/go.sum +++ b/go.sum @@ -294,8 +294,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nspcc-dev/cdn-sdk v0.3.2 h1:R3OCj9yF1PKxPr/HQWTriNOHfsIq/wvSeGYnYkDtVAo= -github.com/nspcc-dev/cdn-sdk v0.3.2/go.mod h1:Wiw3oQjT2F+8ZRt8VnC34ZrSuCDvijyxc1p7bWmcSvk= +github.com/nspcc-dev/cdn-sdk v0.3.3 h1:lAKK6ntvfqlxbn1JN0i4pv3ECFl0jpHVbxamzdTRysg= +github.com/nspcc-dev/cdn-sdk v0.3.3/go.mod h1:wVkJYINe4WtK40uiQRT6OtmbG2zohjl1EM17U5UkgOY= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= @@ -306,8 +306,8 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neofs-api-go v1.22.1 h1:O4uRUM4OxAKKQOebqXIn8uDaER4E5LTEO9Yd7GYkiQA= -github.com/nspcc-dev/neofs-api-go v1.22.1/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-api-go v1.22.2 h1:kNKM7wQMZ23nfFCUgBdkaYaszu4bjgabaUR8nTYBYZQ= +github.com/nspcc-dev/neofs-api-go v1.22.2/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= From dc7aa694b843baa02baaf2ec291b594230db96ef Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 5 Feb 2021 16:51:20 +0300 Subject: [PATCH 095/548] Update Makefile and targets - add `help.mk` to show information about all targets - update Makefile Signed-off-by: Evgeniy Kulikov --- Makefile | 13 ++++++++++++- help.mk | 11 +++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 help.mk diff --git a/Makefile b/Makefile index db412f2..b9aa6c7 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +-include .env +-include help.mk + VERSION ?= "$(shell git describe --tags 2>/dev/null | sed 's/^v//')" GRPC_VERSION=$(shell go list -m google.golang.org/grpc | cut -d " " -f 2) @@ -8,6 +11,8 @@ B=\033[0;1m G=\033[0;92m R=\033[0m +.PHONY: version deps image publish + # Show current version version: @echo "Current version: $(VERSION)-$(GRPC_VERSION)" @@ -21,6 +26,7 @@ deps: @printf "${B}${G}⇒ Store vendor localy${R}: " @go mod vendor && echo OK || (echo fail && exit 2) +# Build docker image image: VERSION?= image: deps @echo "${B}${G}⇒ Build GW docker-image with $(GRPC_VERSION) ${R}" @@ -29,9 +35,14 @@ image: deps -f Dockerfile \ -t $(HUB_IMAGE)-http-gate:$(VERSION) . +# Publish docker image +publish: + @echo "${B}${G}⇒ publish docker image ${R}" + @docker push $(HUB_IMAGE)-http-gate:$(VERSION) + .PHONY: dev -# v1.24.0 v1.25.1 v1.26.0 v1.27.1 +# Build development docker images dev: VERSIONS?=$(GRPC_VERSION) dev: @echo "=> Build multiple images for $(VERSIONS)"; \ diff --git a/help.mk b/help.mk new file mode 100644 index 0000000..c065ec8 --- /dev/null +++ b/help.mk @@ -0,0 +1,11 @@ +.PHONY: help + +# Show this help prompt +help: + @echo ' Usage:' + @echo '' + @echo ' make ' + @echo '' + @echo ' Targets:' + @echo '' + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq From 3b8d684e7c0666d7e2575d5cbb91ffda48ad4117 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 5 Feb 2021 17:13:37 +0300 Subject: [PATCH 096/548] Update README Signed-off-by: Evgeniy Kulikov --- Makefile | 2 +- README.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b9aa6c7..0ec592e 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ R=\033[0m version: @echo "Current version: $(VERSION)-$(GRPC_VERSION)" -# Make sure that all files added to commit +# Check and ensure dependencies deps: @printf "${B}${G}⇒ Ensure vendor${R}: " @go mod tidy -v && echo OK || (echo fail && exit 2) diff --git a/README.md b/README.md index 0a9831b..9d704f3 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,23 @@ NeoFS HTTP Gate is example of tool that provides basic interactions with NeoFS. You can download files from NeoFS Network using NeoFS Gate. +## Notable make targets + +``` + Usage: + + make + + Targets: + + deps Check and ensure dependencies + dev Build development docker images + help Show this help prompt + image Build docker image + publish Publish docker image + version Show current version +``` + ## Install ```go get -u github.com/nspcc-dev/neofs-http-gate``` From d09a1643e1d94b5dc782624f282857c15c67b541 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 12 Feb 2021 12:19:43 +0300 Subject: [PATCH 097/548] Update dependencies - update dependencies - github.com/nspcc-dev/cdn-sdk v0.3.4 - github.com/nspcc-dev/neofs-api-go v1.23.0 - github.com/valyala/fasthttp v1.20.0 Signed-off-by: Evgeniy Kulikov --- go.mod | 6 +++--- go.sum | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index dbfbb98..f8e4e05 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,14 @@ go 1.15 require ( github.com/fasthttp/router v1.3.5 - github.com/nspcc-dev/cdn-sdk v0.3.3 - github.com/nspcc-dev/neofs-api-go v1.22.2 + github.com/nspcc-dev/cdn-sdk v0.3.4 + github.com/nspcc-dev/neofs-api-go v1.23.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 - github.com/valyala/fasthttp v1.19.0 + github.com/valyala/fasthttp v1.20.0 go.uber.org/zap v1.16.0 google.golang.org/grpc v1.35.0 ) diff --git a/go.sum b/go.sum index 2bb3103..b4818b2 100644 --- a/go.sum +++ b/go.sum @@ -294,8 +294,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nspcc-dev/cdn-sdk v0.3.3 h1:lAKK6ntvfqlxbn1JN0i4pv3ECFl0jpHVbxamzdTRysg= -github.com/nspcc-dev/cdn-sdk v0.3.3/go.mod h1:wVkJYINe4WtK40uiQRT6OtmbG2zohjl1EM17U5UkgOY= +github.com/nspcc-dev/cdn-sdk v0.3.4 h1:RtYWuF9xDWrkVwu6sFRWlyZ+ToYM7Y9h8B93Fg4CPEA= +github.com/nspcc-dev/cdn-sdk v0.3.4/go.mod h1:JC4dT16H5HilyZcb8sTxL/TMC1FSEKMuFAqRsmAPoAk= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= @@ -306,8 +306,8 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neofs-api-go v1.22.2 h1:kNKM7wQMZ23nfFCUgBdkaYaszu4bjgabaUR8nTYBYZQ= -github.com/nspcc-dev/neofs-api-go v1.22.2/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-api-go v1.23.0 h1:t4FB5uVY99UkYR0Hiyi1SHjZuqzf4qicw7tf7BBnkHk= +github.com/nspcc-dev/neofs-api-go v1.23.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= @@ -449,6 +449,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.19.0 h1:PfTS4PeH3xDr3WomrDS2ID8lU2GskK1xS3YG6gIpibU= github.com/valyala/fasthttp v1.19.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= +github.com/valyala/fasthttp v1.20.0 h1:olTmcnLQeZrkBc4TVgE/BatTo1NE/IvW050AuD8SW+U= +github.com/valyala/fasthttp v1.20.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= From 462de45e12c0b731e7cce4d031350cab1f0e26de Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 25 Jan 2021 22:36:46 +0300 Subject: [PATCH 098/548] Added POST method to upload files into NeoFS through HTTP Gate - Updated README - Added method to upload files into NeoFS - HTTP Upload Header Filter loaded from settings - Added `HeaderFilter` that filters headers (object attributes) Signed-off-by: Evgeniy Kulikov --- README.md | 5 ++ app.go | 8 ++- filter.go | 86 +++++++++++++++++++++++++++++++ settings.go | 15 +++++- upload.go | 146 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 257 insertions(+), 3 deletions(-) create mode 100644 filter.go create mode 100644 upload.go diff --git a/README.md b/README.md index 9d704f3..c4652ed 100644 --- a/README.md +++ b/README.md @@ -68,4 +68,9 @@ Peers preset: HTTP_GW_PEERS_[N]_ADDRESS = string HTTP_GW_PEERS_[N]_WEIGHT = 0..1 (float) + +Upload Header Table: + +HTTP_GW_UPLOADER_HEADER_[N]_KEY = string - HTTP Header attribute name (except `X-Attribute-`) +HTTP_GW_UPLOADER_HEADER_[N]_VAL = string - NeoFS Object attribute mapping ``` \ No newline at end of file diff --git a/app.go b/app.go index 4b56ac3..57fff2e 100644 --- a/app.go +++ b/app.go @@ -26,6 +26,8 @@ type ( cfg *viper.Viper key *ecdsa.PrivateKey + hdr HeaderFilter + wlog logger.Logger web *fasthttp.Server @@ -74,6 +76,8 @@ func newApp(ctx context.Context, opt ...Option) App { opt[i](a) } + a.hdr = newHeaderFilter(a.log, a.cfg) + a.wlog = logger.GRPC(a.log) if a.cfg.GetBool(cmdVerbose) { @@ -90,7 +94,6 @@ func newApp(ctx context.Context, opt ...Option) App { a.web.WriteBufferSize = a.cfg.GetInt(cfgWebWriteBufferSize) a.web.ReadTimeout = a.cfg.GetDuration(cfgWebReadTimeout) a.web.WriteTimeout = a.cfg.GetDuration(cfgWebWriteTimeout) - a.web.GetOnly = true a.web.DisableHeaderNamesNormalizing = true a.web.NoDefaultServerHeader = true a.web.NoDefaultContentType = true @@ -174,6 +177,9 @@ func (a *app) Serve(ctx context.Context) { r := router.New() r.RedirectTrailingSlash = true + a.log.Info("enabled /put/{cid}") + r.POST("/put/{cid}", a.upload) + a.log.Info("enabled /get/{cid}/{oid}") r.GET("/get/{cid}/{oid}", a.byAddress) diff --git a/filter.go b/filter.go new file mode 100644 index 0000000..9c34015 --- /dev/null +++ b/filter.go @@ -0,0 +1,86 @@ +package main + +import ( + "bytes" + "strconv" + "strings" + + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/spf13/viper" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +type ( + HeaderFilter interface { + Filter(header *fasthttp.RequestHeader) map[string]string + } + + headerFilter struct { + logger *zap.Logger + mapping map[string]string + } +) + +const userAttributeHeader = "X-Attribute-" + +func newHeaderFilter(l *zap.Logger, v *viper.Viper) HeaderFilter { + filter := &headerFilter{ + logger: l, + mapping: make(map[string]string), + } + + for i := 0; ; i++ { + index := strconv.Itoa(i) + key := strings.Join([]string{cfgUploaderHeader, index, cfgUploaderHeaderKey}, ".") + rep := strings.Join([]string{cfgUploaderHeader, index, cfgUploaderHeaderVal}, ".") + + keyValue := v.GetString(key) + repValue := v.GetString(rep) + + if keyValue == "" || repValue == "" { + break + } + + filter.mapping[keyValue] = repValue + + l.Debug("load upload header table value", + zap.String("key", keyValue), + zap.String("val", repValue)) + } + + // Default values + filter.mapping[object.AttributeFileName] = object.AttributeFileName + filter.mapping[object.AttributeTimestamp] = object.AttributeTimestamp + + return filter +} + +func (h *headerFilter) Filter(header *fasthttp.RequestHeader) map[string]string { + result := make(map[string]string) + prefix := []byte(userAttributeHeader) + + header.VisitAll(func(key, val []byte) { + if len(key) == 0 || len(val) == 0 { + return + } else if !bytes.HasPrefix(key, prefix) { + return + } else if key = bytes.TrimPrefix(key, prefix); len(key) == 0 { + return + } else if name, ok := h.mapping[string(key)]; ok { + result[name] = string(val) + + h.logger.Debug("add attribute to result object", + zap.String("key", name), + zap.String("val", string(val))) + + return + } + + h.logger.Debug("ignore attribute", + zap.String("key", string(key)), + zap.String("val", string(val))) + }) + + return result +} diff --git a/settings.go b/settings.go index c781056..2917a18 100644 --- a/settings.go +++ b/settings.go @@ -16,8 +16,7 @@ import ( type empty int const ( - devNull = empty(0) - generated = "generated" + devNull = empty(0) defaultRebalanceTimer = 15 * time.Second defaultRequestTimeout = 15 * time.Second @@ -54,6 +53,11 @@ const ( cfgLoggerSamplingInitial = "logger.sampling.initial" cfgLoggerSamplingThereafter = "logger.sampling.thereafter" + // Uploader Header + cfgUploaderHeader = "uploader_header" + cfgUploaderHeaderKey = "key" + cfgUploaderHeaderVal = "val" + // Peers cfgPeers = "peers" @@ -179,6 +183,13 @@ func settings() *viper.Viper { fmt.Printf("%s_%s_[N]_ADDRESS = string\n", Prefix, strings.ToUpper(cfgPeers)) fmt.Printf("%s_%s_[N]_WEIGHT = 0..1 (float)\n", Prefix, strings.ToUpper(cfgPeers)) + fmt.Println() + fmt.Println("Upload Header Table:") + fmt.Println() + + fmt.Printf("%s_%s_[N]_%s = string\n", Prefix, strings.ToUpper(cfgUploaderHeader), strings.ToUpper(cfgUploaderHeaderKey)) + fmt.Printf("%s_%s_[N]_%s = string\n", Prefix, strings.ToUpper(cfgUploaderHeader), strings.ToUpper(cfgUploaderHeaderVal)) + os.Exit(0) case version != nil && *version: fmt.Printf("NeoFS HTTP Gateway %s (%s)\n", Version, Build) diff --git a/upload.go b/upload.go new file mode 100644 index 0000000..128c3eb --- /dev/null +++ b/upload.go @@ -0,0 +1,146 @@ +package main + +import ( + "encoding/json" + "io" + "io/ioutil" + "mime/multipart" + "os" + "strconv" + "time" + + sdk "github.com/nspcc-dev/cdn-sdk" + "github.com/nspcc-dev/neofs-api-go/pkg/container" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +type putResponse struct { + OID string `json:"object_id"` + CID string `json:"container_id"` +} + +func newPutResponse(addr *object.Address) *putResponse { + return &putResponse{ + OID: addr.ObjectID().String(), + CID: addr.ContainerID().String(), + } +} + +func (pr *putResponse) Encode(w io.Writer) error { + enc := json.NewEncoder(w) + enc.SetIndent("", "\t") + return enc.Encode(pr) +} + +func (a *app) upload(c *fasthttp.RequestCtx) { + var ( + err error + name string + tmp *os.File + addr *object.Address + form *multipart.Form + cid = container.NewID() + sCID, _ = c.UserValue("cid").(string) + + log = a.log.With(zap.String("cid", sCID)) + ) + + if err = cid.Parse(sCID); err != nil { + log.Error("wrong container id", zap.Error(err)) + c.Error("wrong container id", fasthttp.StatusBadRequest) + return + } + + if tmp, err = ioutil.TempFile("", "http-gate-upload-*"); err != nil { + log.Error("could not prepare temporary file", zap.Error(err)) + c.Error("could not prepare temporary file", fasthttp.StatusBadRequest) + return + } + + defer func() { + tmpName := tmp.Name() + + log.Debug("close temporary file", zap.Error(tmp.Close())) + log.Debug("remove temporary file", zap.Error(os.RemoveAll(tmpName))) + }() + + if form, err = c.MultipartForm(); err != nil { + log.Error("could not receive multipart form", zap.Error(err)) + c.Error("could not receive multipart form: "+err.Error(), fasthttp.StatusBadRequest) + + return + } else if ln := len(form.File); ln != 1 { + log.Error("received multipart form with more then one file", zap.Int("count", ln)) + c.Error("received multipart form with more then one file", fasthttp.StatusBadRequest) + + return + } + + for _, file := range form.File { + if ln := len(file); ln != 1 { + log.Error("received multipart form file should contains one record", zap.Int("count", ln)) + c.Error("received multipart form file should contains one record", fasthttp.StatusBadRequest) + + return + } + + name = file[0].Filename + + if err = fasthttp.SaveMultipartFile(file[0], tmp.Name()); err != nil { + log.Error("could not store uploaded file into temporary", zap.Error(err)) + c.Error("could not store uploaded file into temporary", fasthttp.StatusBadRequest) + + return + } + } + + filtered := a.hdr.Filter(&c.Request.Header) + attributes := make([]*object.Attribute, 0, len(filtered)) + + for key, val := range filtered { + attribute := object.NewAttribute() + attribute.SetKey(key) + attribute.SetValue(val) + + attributes = append(attributes, attribute) + } + + // Attribute FileName wasn't set from header + if _, ok := filtered[object.AttributeFileName]; ok { + filename := object.NewAttribute() + filename.SetKey(object.AttributeFileName) + filename.SetValue(name) + + attributes = append(attributes, filename) + } + + // Attribute Timestamp wasn't set from header + if _, ok := filtered[object.AttributeTimestamp]; ok { + timestamp := object.NewAttribute() + timestamp.SetKey(object.AttributeTimestamp) + timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) + + attributes = append(attributes, timestamp) + } + + raw := object.NewRaw() + raw.SetContainerID(cid) + raw.SetOwnerID(a.cli.Owner()) // should be various: from sdk / BearerToken + raw.SetAttributes(attributes...) + + if addr, err = a.cli.Object().Put(c, raw.Object(), sdk.WithPutReader(tmp)); err != nil { + log.Error("could not store file in NeoFS", zap.Error(err)) + c.Error("could not store file in NeoFS", fasthttp.StatusBadRequest) + + return + } else if err = newPutResponse(addr).Encode(c); err != nil { + log.Error("could not prepare response", zap.Error(err)) + c.Error("could not prepare response", fasthttp.StatusBadRequest) + + return + } + + c.Response.SetStatusCode(fasthttp.StatusOK) +} From 0346db462bbbad63631d39623ea75ebf10f8ef7f Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 26 Jan 2021 11:35:05 +0300 Subject: [PATCH 099/548] Add cleanup multipart form on defer Signed-off-by: Evgeniy Kulikov --- upload.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/upload.go b/upload.go index 128c3eb..f5e4324 100644 --- a/upload.go +++ b/upload.go @@ -64,6 +64,12 @@ func (a *app) upload(c *fasthttp.RequestCtx) { log.Debug("close temporary file", zap.Error(tmp.Close())) log.Debug("remove temporary file", zap.Error(os.RemoveAll(tmpName))) + + if form == nil { + return + } + + log.Debug("cleanup multipart form", zap.Error(form.RemoveAll())) }() if form, err = c.MultipartForm(); err != nil { From ace31ceefd95b8e8cc22ee3c4c459336d4ab87bf Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 26 Jan 2021 11:43:40 +0300 Subject: [PATCH 100/548] Add `application/json` Content-Type header Signed-off-by: Evgeniy Kulikov --- upload.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/upload.go b/upload.go index f5e4324..0850cf6 100644 --- a/upload.go +++ b/upload.go @@ -21,6 +21,8 @@ type putResponse struct { CID string `json:"container_id"` } +const jsonHeader = "application/json; charset=UTF-8" + func newPutResponse(addr *object.Address) *putResponse { return &putResponse{ OID: addr.ObjectID().String(), @@ -149,4 +151,5 @@ func (a *app) upload(c *fasthttp.RequestCtx) { } c.Response.SetStatusCode(fasthttp.StatusOK) + c.Response.Header.SetContentType(jsonHeader) } From 3cbd4dbd097a39f3f086b8984de2e695a87d466f Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 26 Jan 2021 12:40:01 +0300 Subject: [PATCH 101/548] Fixes after review - Maybe it could be rephrased to "attribute name prefixed with X-Attribute-" - Maybe it worth renaming to userAttributeHeaderPrefix - Requirement for having only one file uploaded at a time has to be reflected in documentation. - Maybe this should be reflected in doc - I'm not sure if missing timestamp attribute should be forced on gateway level. Normally it should be set by app and there should be a way not to set it. Signed-off-by: Evgeniy Kulikov --- README.md | 17 +++++++++++++++-- app.go | 4 ++++ filter.go | 4 ++-- settings.go | 10 +++++++--- upload.go | 4 ++-- 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c4652ed..d9f875b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # NeoFS HTTP Gate NeoFS HTTP Gate is example of tool that provides basic interactions with NeoFS. -You can download files from NeoFS Network using NeoFS Gate. + +- you can download one file per request from NeoFS Network using NeoFS Gate +- you can upload one file per request into NeoFS Network using NeoFS Gate ## Notable make targets @@ -24,6 +26,11 @@ You can download files from NeoFS Network using NeoFS Gate. ```go get -u github.com/nspcc-dev/neofs-http-gate``` +## File uploading behaviors + +- you can upload on file per request +- if `FileName` not provided by Header attributes, multipart/form filename will be used instead + ## Configuration ``` @@ -64,6 +71,8 @@ of Timeout and if no activity is seen even after that the connection is closed HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=Bool - if true, client sends keepalive pings even with no active RPCs. If false, when there are no active RPCs, Time and Timeout will be ignored and no keepalive pings will be sent. +HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=bool - enable/disable adding current timestamp attribute when object uploads + Peers preset: HTTP_GW_PEERS_[N]_ADDRESS = string @@ -71,6 +80,10 @@ HTTP_GW_PEERS_[N]_WEIGHT = 0..1 (float) Upload Header Table: -HTTP_GW_UPLOADER_HEADER_[N]_KEY = string - HTTP Header attribute name (except `X-Attribute-`) +HTTP_GW_UPLOADER_HEADER_[N]_KEY = string - HTTP Header attribute name prefixed with `X-Attribute-` HTTP_GW_UPLOADER_HEADER_[N]_VAL = string - NeoFS Object attribute mapping + +# By default we had next headers: +- FileName - to set object filename attribute +- Timestamp - to set object timestamp attribute ``` \ No newline at end of file diff --git a/app.go b/app.go index 57fff2e..f565793 100644 --- a/app.go +++ b/app.go @@ -33,6 +33,8 @@ type ( jobDone chan struct{} webDone chan struct{} + + enableDefaultTimestamp bool } App interface { @@ -78,6 +80,8 @@ func newApp(ctx context.Context, opt ...Option) App { a.hdr = newHeaderFilter(a.log, a.cfg) + a.enableDefaultTimestamp = a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) + a.wlog = logger.GRPC(a.log) if a.cfg.GetBool(cmdVerbose) { diff --git a/filter.go b/filter.go index 9c34015..2f12eb5 100644 --- a/filter.go +++ b/filter.go @@ -22,7 +22,7 @@ type ( } ) -const userAttributeHeader = "X-Attribute-" +const userAttributeHeaderPrefix = "X-Attribute-" func newHeaderFilter(l *zap.Logger, v *viper.Viper) HeaderFilter { filter := &headerFilter{ @@ -58,7 +58,7 @@ func newHeaderFilter(l *zap.Logger, v *viper.Viper) HeaderFilter { func (h *headerFilter) Filter(header *fasthttp.RequestHeader) map[string]string { result := make(map[string]string) - prefix := []byte(userAttributeHeader) + prefix := []byte(userAttributeHeaderPrefix) header.VisitAll(func(key, val []byte) { if len(key) == 0 || len(val) == 0 { diff --git a/settings.go b/settings.go index 2917a18..1775569 100644 --- a/settings.go +++ b/settings.go @@ -54,9 +54,10 @@ const ( cfgLoggerSamplingThereafter = "logger.sampling.thereafter" // Uploader Header - cfgUploaderHeader = "uploader_header" - cfgUploaderHeaderKey = "key" - cfgUploaderHeaderVal = "val" + cfgUploaderHeader = "uploader_header" + cfgUploaderHeaderKey = "key" + cfgUploaderHeaderVal = "val" + cfgUploaderHeaderEnableDefaultTimestamp = "upload_header.use_default_timestamp" // Peers cfgPeers = "peers" @@ -144,6 +145,9 @@ func settings() *viper.Viper { v.SetDefault(cfgWebWriteTimeout, time.Minute) v.SetDefault(cfgWebConnectionPerHost, 10) + // upload header + v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) + if err := v.BindPFlags(flags); err != nil { panic(err) } diff --git a/upload.go b/upload.go index 0850cf6..b1330fc 100644 --- a/upload.go +++ b/upload.go @@ -124,8 +124,8 @@ func (a *app) upload(c *fasthttp.RequestCtx) { attributes = append(attributes, filename) } - // Attribute Timestamp wasn't set from header - if _, ok := filtered[object.AttributeTimestamp]; ok { + // Attribute Timestamp wasn't set from header and enabled by settings + if _, ok := filtered[object.AttributeTimestamp]; ok && a.enableDefaultTimestamp { timestamp := object.NewAttribute() timestamp.SetKey(object.AttributeTimestamp) timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) From ec70bfa4cc54a5e190ce40bdaab761f54f3f5d46 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 26 Jan 2021 18:36:53 +0300 Subject: [PATCH 102/548] Fixes after review - It's confusing. If there is no difference, I suggest having the route named after the protocol verb or resulting handler. So it should be either post or upload. - Don't you find that it would be more understandable without else ifs? - Why do we need a temporary file here? - etc Signed-off-by: Evgeniy Kulikov --- app.go | 4 ++-- filter.go | 18 ++++++++++++++--- upload.go | 60 ++++++++++++++++++++++++++++++------------------------- 3 files changed, 50 insertions(+), 32 deletions(-) diff --git a/app.go b/app.go index f565793..6d666eb 100644 --- a/app.go +++ b/app.go @@ -181,8 +181,8 @@ func (a *app) Serve(ctx context.Context) { r := router.New() r.RedirectTrailingSlash = true - a.log.Info("enabled /put/{cid}") - r.POST("/put/{cid}", a.upload) + a.log.Info("enabled /upload/{cid}") + r.POST("/upload/{cid}", a.upload) a.log.Info("enabled /get/{cid}/{oid}") r.GET("/get/{cid}/{oid}", a.byAddress) diff --git a/filter.go b/filter.go index 2f12eb5..7e13766 100644 --- a/filter.go +++ b/filter.go @@ -61,13 +61,24 @@ func (h *headerFilter) Filter(header *fasthttp.RequestHeader) map[string]string prefix := []byte(userAttributeHeaderPrefix) header.VisitAll(func(key, val []byte) { + // checks that key and val not empty if len(key) == 0 || len(val) == 0 { return - } else if !bytes.HasPrefix(key, prefix) { + } + + // checks that key has attribute prefix + if !bytes.HasPrefix(key, prefix) { return - } else if key = bytes.TrimPrefix(key, prefix); len(key) == 0 { + } + + // checks that after removing attribute prefix we had not empty key + if key = bytes.TrimPrefix(key, prefix); len(key) == 0 { return - } else if name, ok := h.mapping[string(key)]; ok { + } + + // checks mapping table and if we found record store it + // at resulting hashmap + if name, ok := h.mapping[string(key)]; ok { result[name] = string(val) h.logger.Debug("add attribute to result object", @@ -77,6 +88,7 @@ func (h *headerFilter) Filter(header *fasthttp.RequestHeader) map[string]string return } + // otherwise inform that attribute will be ignored h.logger.Debug("ignore attribute", zap.String("key", string(key)), zap.String("val", string(val))) diff --git a/upload.go b/upload.go index b1330fc..6d72a61 100644 --- a/upload.go +++ b/upload.go @@ -3,9 +3,7 @@ package main import ( "encoding/json" "io" - "io/ioutil" "mime/multipart" - "os" "strconv" "time" @@ -40,7 +38,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { var ( err error name string - tmp *os.File + tmp io.Reader addr *object.Address form *multipart.Form cid = container.NewID() @@ -55,50 +53,51 @@ func (a *app) upload(c *fasthttp.RequestCtx) { return } - if tmp, err = ioutil.TempFile("", "http-gate-upload-*"); err != nil { - log.Error("could not prepare temporary file", zap.Error(err)) - c.Error("could not prepare temporary file", fasthttp.StatusBadRequest) - return - } - defer func() { - tmpName := tmp.Name() - - log.Debug("close temporary file", zap.Error(tmp.Close())) - log.Debug("remove temporary file", zap.Error(os.RemoveAll(tmpName))) + // if temporary reader can be closed - close it + if closer := tmp.(io.Closer); closer != nil { + log.Debug("close temporary multipart/form file", zap.Error(closer.Close())) + } if form == nil { return } - log.Debug("cleanup multipart form", zap.Error(form.RemoveAll())) + log.Debug("cleanup multipart/form", zap.Error(form.RemoveAll())) }() + // tries to receive multipart/form or throw error if form, err = c.MultipartForm(); err != nil { - log.Error("could not receive multipart form", zap.Error(err)) - c.Error("could not receive multipart form: "+err.Error(), fasthttp.StatusBadRequest) + log.Error("could not receive multipart/form", zap.Error(err)) + c.Error("could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return - } else if ln := len(form.File); ln != 1 { - log.Error("received multipart form with more then one file", zap.Int("count", ln)) - c.Error("received multipart form with more then one file", fasthttp.StatusBadRequest) + } + + // checks that received multipart/form contains only one `file` per request + if ln := len(form.File); ln != 1 { + log.Error("received multipart/form with more then one file", zap.Int("count", ln)) + c.Error("received multipart/form with more then one file", fasthttp.StatusBadRequest) return } for _, file := range form.File { + // because multipart/form can contains multiple FileHeader records + // we should check that we have only one per request or throw error if ln := len(file); ln != 1 { - log.Error("received multipart form file should contains one record", zap.Int("count", ln)) - c.Error("received multipart form file should contains one record", fasthttp.StatusBadRequest) + log.Error("received multipart/form file should contains one record", zap.Int("count", ln)) + c.Error("received multipart/form file should contains one record", fasthttp.StatusBadRequest) return } name = file[0].Filename - if err = fasthttp.SaveMultipartFile(file[0], tmp.Name()); err != nil { - log.Error("could not store uploaded file into temporary", zap.Error(err)) - c.Error("could not store uploaded file into temporary", fasthttp.StatusBadRequest) + // opens multipart/form file to work within or throw error + if tmp, err = file[0].Open(); err != nil { + log.Error("could not prepare uploaded file", zap.Error(err)) + c.Error("could not prepare uploaded file", fasthttp.StatusBadRequest) return } @@ -107,6 +106,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { filtered := a.hdr.Filter(&c.Request.Header) attributes := make([]*object.Attribute, 0, len(filtered)) + // prepares attributes from filtered headers for key, val := range filtered { attribute := object.NewAttribute() attribute.SetKey(key) @@ -115,7 +115,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { attributes = append(attributes, attribute) } - // Attribute FileName wasn't set from header + // sets FileName attribute if it wasn't set from header if _, ok := filtered[object.AttributeFileName]; ok { filename := object.NewAttribute() filename.SetKey(object.AttributeFileName) @@ -124,7 +124,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { attributes = append(attributes, filename) } - // Attribute Timestamp wasn't set from header and enabled by settings + // sets Timestamp attribute if it wasn't set from header and enabled by settings if _, ok := filtered[object.AttributeTimestamp]; ok && a.enableDefaultTimestamp { timestamp := object.NewAttribute() timestamp.SetKey(object.AttributeTimestamp) @@ -133,23 +133,29 @@ func (a *app) upload(c *fasthttp.RequestCtx) { attributes = append(attributes, timestamp) } + // prepares new object and fill it raw := object.NewRaw() raw.SetContainerID(cid) raw.SetOwnerID(a.cli.Owner()) // should be various: from sdk / BearerToken raw.SetAttributes(attributes...) + // tries to put file into NeoFS or throw error if addr, err = a.cli.Object().Put(c, raw.Object(), sdk.WithPutReader(tmp)); err != nil { log.Error("could not store file in NeoFS", zap.Error(err)) c.Error("could not store file in NeoFS", fasthttp.StatusBadRequest) return - } else if err = newPutResponse(addr).Encode(c); err != nil { + } + + // tries to return response, otherwise, if something went wrong throw error + if err = newPutResponse(addr).Encode(c); err != nil { log.Error("could not prepare response", zap.Error(err)) c.Error("could not prepare response", fasthttp.StatusBadRequest) return } + // reports status code and content type c.Response.SetStatusCode(fasthttp.StatusOK) c.Response.Header.SetContentType(jsonHeader) } From 71999a796d675ed29a1497f17633d0c503594cb4 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 28 Jan 2021 16:48:45 +0300 Subject: [PATCH 103/548] Fix logic with set attributes if not set from header - if NOT set Filename from header - set from multipart/form - if NOT set Timestamp from header and enabled by settings - set current timestamp Signed-off-by: Evgeniy Kulikov --- upload.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/upload.go b/upload.go index 6d72a61..12111b0 100644 --- a/upload.go +++ b/upload.go @@ -116,7 +116,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { } // sets FileName attribute if it wasn't set from header - if _, ok := filtered[object.AttributeFileName]; ok { + if _, ok := filtered[object.AttributeFileName]; !ok { filename := object.NewAttribute() filename.SetKey(object.AttributeFileName) filename.SetValue(name) @@ -125,7 +125,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { } // sets Timestamp attribute if it wasn't set from header and enabled by settings - if _, ok := filtered[object.AttributeTimestamp]; ok && a.enableDefaultTimestamp { + if _, ok := filtered[object.AttributeTimestamp]; !ok && a.enableDefaultTimestamp { timestamp := object.NewAttribute() timestamp.SetKey(object.AttributeTimestamp) timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) From cbaf9e61428772fd53effd52e5a7bf7ed0abaa43 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Wed, 3 Feb 2021 16:01:30 +0300 Subject: [PATCH 104/548] Fixes after review After discussion, we decided to simplify attribute translation for now. Full translation requires support for UTF-8 values encoded according to RFC 8187/7230, this can be clarified and implemented in further PRs. 1. Remove translation tables 2. Use simple translation rules only, for now ignoring UTF-8 keys and values - `X-Attribute-NEOFS-` prefixed headers considered well-known system attributes - System Attribute key is uppercased - System Attribute key's `-` translates to `_` - `X-Attribute-` prefixed headers considered regular object attributes - Normal attribute key is a result of http header prefix trimming - Value string should be left as is, for now ``` HTTP: X-Attribute-NEOFS-Expiration-Epoch: 123 X-Attribute-FileName: cat.jpg NeoFS: __NEOFS__EXPIRATION_EPOCH: "123" FileName: "cat.jpg" ``` Signed-off-by: Evgeniy Kulikov --- README.md | 9 ------ app.go | 4 --- filter.go | 88 ++++++++++++++++++----------------------------------- settings.go | 10 ------ upload.go | 2 +- 5 files changed, 30 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index d9f875b..f4b30d4 100644 --- a/README.md +++ b/README.md @@ -77,13 +77,4 @@ Peers preset: HTTP_GW_PEERS_[N]_ADDRESS = string HTTP_GW_PEERS_[N]_WEIGHT = 0..1 (float) - -Upload Header Table: - -HTTP_GW_UPLOADER_HEADER_[N]_KEY = string - HTTP Header attribute name prefixed with `X-Attribute-` -HTTP_GW_UPLOADER_HEADER_[N]_VAL = string - NeoFS Object attribute mapping - -# By default we had next headers: -- FileName - to set object filename attribute -- Timestamp - to set object timestamp attribute ``` \ No newline at end of file diff --git a/app.go b/app.go index 6d666eb..ca3d249 100644 --- a/app.go +++ b/app.go @@ -26,8 +26,6 @@ type ( cfg *viper.Viper key *ecdsa.PrivateKey - hdr HeaderFilter - wlog logger.Logger web *fasthttp.Server @@ -78,8 +76,6 @@ func newApp(ctx context.Context, opt ...Option) App { opt[i](a) } - a.hdr = newHeaderFilter(a.log, a.cfg) - a.enableDefaultTimestamp = a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) a.wlog = logger.GRPC(a.log) diff --git a/filter.go b/filter.go index 7e13766..7d8b24d 100644 --- a/filter.go +++ b/filter.go @@ -2,63 +2,33 @@ package main import ( "bytes" - "strconv" - "strings" - "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" ) -type ( - HeaderFilter interface { - Filter(header *fasthttp.RequestHeader) map[string]string - } +const ( + userAttributeHeaderPrefix = "X-Attribute-" + neofsAttributeHeaderPrefix = "NEOFS-" - headerFilter struct { - logger *zap.Logger - mapping map[string]string - } + systemAttributePrefix = "__NEOFS__" ) -const userAttributeHeaderPrefix = "X-Attribute-" +func systemTranslator(key []byte) []byte { + // replace `NEOFS-` with `__NEOFS__` + key = bytes.Replace(key, []byte(neofsAttributeHeaderPrefix), []byte(systemAttributePrefix), 1) -func newHeaderFilter(l *zap.Logger, v *viper.Viper) HeaderFilter { - filter := &headerFilter{ - logger: l, - mapping: make(map[string]string), - } + // replace `-` with `_` + key = bytes.ReplaceAll(key, []byte("-"), []byte("_")) - for i := 0; ; i++ { - index := strconv.Itoa(i) - key := strings.Join([]string{cfgUploaderHeader, index, cfgUploaderHeaderKey}, ".") - rep := strings.Join([]string{cfgUploaderHeader, index, cfgUploaderHeaderVal}, ".") - - keyValue := v.GetString(key) - repValue := v.GetString(rep) - - if keyValue == "" || repValue == "" { - break - } - - filter.mapping[keyValue] = repValue - - l.Debug("load upload header table value", - zap.String("key", keyValue), - zap.String("val", repValue)) - } - - // Default values - filter.mapping[object.AttributeFileName] = object.AttributeFileName - filter.mapping[object.AttributeTimestamp] = object.AttributeTimestamp - - return filter + // replace with uppercase + return bytes.ToUpper(key) } -func (h *headerFilter) Filter(header *fasthttp.RequestHeader) map[string]string { +func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]string { result := make(map[string]string) prefix := []byte(userAttributeHeaderPrefix) + system := []byte(neofsAttributeHeaderPrefix) header.VisitAll(func(key, val []byte) { // checks that key and val not empty @@ -71,27 +41,27 @@ func (h *headerFilter) Filter(header *fasthttp.RequestHeader) map[string]string return } - // checks that after removing attribute prefix we had not empty key - if key = bytes.TrimPrefix(key, prefix); len(key) == 0 { + // removing attribute prefix + key = bytes.TrimPrefix(key, prefix) + + // checks that it's a system NeoFS header + if bytes.HasPrefix(key, system) { + key = systemTranslator(key) + } + + // checks that attribute key not empty + if len(key) == 0 { return } - // checks mapping table and if we found record store it - // at resulting hashmap - if name, ok := h.mapping[string(key)]; ok { - result[name] = string(val) + // make string representation of key / val + k, v := string(key), string(val) - h.logger.Debug("add attribute to result object", - zap.String("key", name), - zap.String("val", string(val))) + result[k] = v - return - } - - // otherwise inform that attribute will be ignored - h.logger.Debug("ignore attribute", - zap.String("key", string(key)), - zap.String("val", string(val))) + l.Debug("add attribute to result object", + zap.String("key", k), + zap.String("val", v)) }) return result diff --git a/settings.go b/settings.go index 1775569..02fcde8 100644 --- a/settings.go +++ b/settings.go @@ -54,9 +54,6 @@ const ( cfgLoggerSamplingThereafter = "logger.sampling.thereafter" // Uploader Header - cfgUploaderHeader = "uploader_header" - cfgUploaderHeaderKey = "key" - cfgUploaderHeaderVal = "val" cfgUploaderHeaderEnableDefaultTimestamp = "upload_header.use_default_timestamp" // Peers @@ -187,13 +184,6 @@ func settings() *viper.Viper { fmt.Printf("%s_%s_[N]_ADDRESS = string\n", Prefix, strings.ToUpper(cfgPeers)) fmt.Printf("%s_%s_[N]_WEIGHT = 0..1 (float)\n", Prefix, strings.ToUpper(cfgPeers)) - fmt.Println() - fmt.Println("Upload Header Table:") - fmt.Println() - - fmt.Printf("%s_%s_[N]_%s = string\n", Prefix, strings.ToUpper(cfgUploaderHeader), strings.ToUpper(cfgUploaderHeaderKey)) - fmt.Printf("%s_%s_[N]_%s = string\n", Prefix, strings.ToUpper(cfgUploaderHeader), strings.ToUpper(cfgUploaderHeaderVal)) - os.Exit(0) case version != nil && *version: fmt.Printf("NeoFS HTTP Gateway %s (%s)\n", Version, Build) diff --git a/upload.go b/upload.go index 12111b0..767ee2e 100644 --- a/upload.go +++ b/upload.go @@ -103,7 +103,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { } } - filtered := a.hdr.Filter(&c.Request.Header) + filtered := filterHeaders(a.log, &c.Request.Header) attributes := make([]*object.Attribute, 0, len(filtered)) // prepares attributes from filtered headers From a428a0b1b3dd49f4bb5cd7f04d4b1b0017327c2d Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 12 Feb 2021 15:24:52 +0300 Subject: [PATCH 105/548] fix conflicts --- README.md | 9 +++++++++ app.go | 11 +++++++++++ settings.go | 37 +++++++++++++++++++++++++++++++------ upload.go | 2 +- 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f4b30d4..73aaeda 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,15 @@ If false, when there are no active RPCs, Time and Timeout will be ignored and no HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=bool - enable/disable adding current timestamp attribute when object uploads +HTTP_GW_WEB_READ_BUFFER_SIZE=4096 - per-connection buffer size for requests' reading +HTTP_GW_WEB_READ_TIMEOUT=15s - an amount of time allowed to read the full request including body +HTTP_GW_WEB_WRITE_BUFFER_SIZE=4096 - per-connection buffer size for responses' writing +HTTP_GW_WEB_WRITE_TIMEOUT=1m0s - maximum duration before timing out writes of the response +HTTP_GW_WEB_STREAM_REQUEST_BODY=true - enables request body streaming, and calls the handler sooner when given + body is larger then the current limit +HTTP_GW_WEB_MAX_REQUEST_BODY_SIZE=4194304 - maximum request body size, server rejects requests with bodies exceeding + this limit + Peers preset: HTTP_GW_PEERS_[N]_ADDRESS = string diff --git a/app.go b/app.go index ca3d249..34a1492 100644 --- a/app.go +++ b/app.go @@ -97,6 +97,17 @@ func newApp(ctx context.Context, opt ...Option) App { a.web.DisableHeaderNamesNormalizing = true a.web.NoDefaultServerHeader = true a.web.NoDefaultContentType = true + a.web.MaxRequestBodySize = a.cfg.GetInt(cfgWebMaxRequestBodySize) + + // FIXME don't work with StreamRequestBody, + // some bugs with readMultipartForm + // a.web.DisablePreParseMultipartForm = true + + // body streaming + // TODO should be replaced in future with + // + // a.web.StreamRequestBody = v.GetBool(cfgWebStreamRequestBody) + checkAndEnableStreaming(a.log, a.cfg, a.web) // -- -- -- -- -- -- -- -- -- -- connections := make(map[string]float64) diff --git a/settings.go b/settings.go index 02fcde8..b2dac09 100644 --- a/settings.go +++ b/settings.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "reflect" "sort" "strconv" "strings" @@ -11,6 +12,8 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" + "github.com/valyala/fasthttp" + "go.uber.org/zap" ) type empty int @@ -33,11 +36,12 @@ const ( cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream" // Web - cfgWebReadBufferSize = "web.read_buffer_size" - cfgWebWriteBufferSize = "web.write_buffer_size" - cfgWebReadTimeout = "web.read_timeout" - cfgWebWriteTimeout = "web.write_timeout" - cfgWebConnectionPerHost = "web.connection_per_host" + cfgWebReadBufferSize = "web.read_buffer_size" + cfgWebWriteBufferSize = "web.write_buffer_size" + cfgWebReadTimeout = "web.read_timeout" + cfgWebWriteTimeout = "web.write_timeout" + cfgWebStreamRequestBody = "web.stream_request_body" + cfgWebMaxRequestBodySize = "web.max_request_body_size" // Timeouts cfgConTimeout = "connect_timeout" @@ -86,6 +90,26 @@ var ignore = map[string]struct{}{ func (empty) Read([]byte) (int, error) { return 0, io.EOF } +// checkAndEnableStreaming is temporary shim, should be used before +// `StreamRequestBody` is not merged in fasthttp master +// TODO should be removed in future +func checkAndEnableStreaming(l *zap.Logger, v *viper.Viper, i interface{}) { + vi := reflect.ValueOf(i) + + if vi.Type().Kind() != reflect.Ptr { + return + } + + field := vi.Elem().FieldByName("StreamRequestBody") + if !field.IsValid() || field.Kind() != reflect.Bool { + l.Warn("stream request body not supported") + + return + } + + field.SetBool(v.GetBool(cfgWebStreamRequestBody)) +} + func settings() *viper.Viper { v := viper.New() v.AutomaticEnv() @@ -140,7 +164,8 @@ func settings() *viper.Viper { v.SetDefault(cfgWebWriteBufferSize, 4096) v.SetDefault(cfgWebReadTimeout, time.Second*15) v.SetDefault(cfgWebWriteTimeout, time.Minute) - v.SetDefault(cfgWebConnectionPerHost, 10) + v.SetDefault(cfgWebStreamRequestBody, true) + v.SetDefault(cfgWebMaxRequestBodySize, fasthttp.DefaultMaxRequestBodySize) // upload header v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) diff --git a/upload.go b/upload.go index 767ee2e..38d53e9 100644 --- a/upload.go +++ b/upload.go @@ -55,7 +55,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { defer func() { // if temporary reader can be closed - close it - if closer := tmp.(io.Closer); closer != nil { + if closer, ok := tmp.(io.Closer); ok && closer != nil { log.Debug("close temporary multipart/form file", zap.Error(closer.Close())) } From 3b8bf3017d823472f4dad2edf9e6e5c246a8081f Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 12 Feb 2021 18:48:52 +0300 Subject: [PATCH 106/548] Migrate to fasthttp v1.20.0 Signed-off-by: Evgeniy Kulikov --- app.go | 7 ++----- settings.go | 22 ---------------------- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/app.go b/app.go index 34a1492..8e4a961 100644 --- a/app.go +++ b/app.go @@ -101,13 +101,10 @@ func newApp(ctx context.Context, opt ...Option) App { // FIXME don't work with StreamRequestBody, // some bugs with readMultipartForm + // https://github.com/valyala/fasthttp/issues/968 // a.web.DisablePreParseMultipartForm = true - // body streaming - // TODO should be replaced in future with - // - // a.web.StreamRequestBody = v.GetBool(cfgWebStreamRequestBody) - checkAndEnableStreaming(a.log, a.cfg, a.web) + a.web.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- connections := make(map[string]float64) diff --git a/settings.go b/settings.go index b2dac09..206fd0a 100644 --- a/settings.go +++ b/settings.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "os" - "reflect" "sort" "strconv" "strings" @@ -13,7 +12,6 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/valyala/fasthttp" - "go.uber.org/zap" ) type empty int @@ -90,26 +88,6 @@ var ignore = map[string]struct{}{ func (empty) Read([]byte) (int, error) { return 0, io.EOF } -// checkAndEnableStreaming is temporary shim, should be used before -// `StreamRequestBody` is not merged in fasthttp master -// TODO should be removed in future -func checkAndEnableStreaming(l *zap.Logger, v *viper.Viper, i interface{}) { - vi := reflect.ValueOf(i) - - if vi.Type().Kind() != reflect.Ptr { - return - } - - field := vi.Elem().FieldByName("StreamRequestBody") - if !field.IsValid() || field.Kind() != reflect.Bool { - l.Warn("stream request body not supported") - - return - } - - field.SetBool(v.GetBool(cfgWebStreamRequestBody)) -} - func settings() *viper.Viper { v := viper.New() v.AutomaticEnv() From 3f635a018aa043ba465a52b9c2a9a009b1285914 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Sat, 13 Feb 2021 19:17:01 +0300 Subject: [PATCH 107/548] Refactoring uploading - DisablePreParseMultipartForm = true - used `fetchMultipartFile` method instead of `MultipartForm` - don't store temporary files, only streaming Signed-off-by: Evgeniy Kulikov --- app.go | 2 +- multipart.go | 41 ++++++++++++++++++++++++++++++++++++++++ upload.go | 53 ++++++++++------------------------------------------ 3 files changed, 52 insertions(+), 44 deletions(-) create mode 100644 multipart.go diff --git a/app.go b/app.go index 8e4a961..6c70b90 100644 --- a/app.go +++ b/app.go @@ -102,7 +102,7 @@ func newApp(ctx context.Context, opt ...Option) App { // FIXME don't work with StreamRequestBody, // some bugs with readMultipartForm // https://github.com/valyala/fasthttp/issues/968 - // a.web.DisablePreParseMultipartForm = true + a.web.DisablePreParseMultipartForm = true a.web.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- diff --git a/multipart.go b/multipart.go new file mode 100644 index 0000000..f12442f --- /dev/null +++ b/multipart.go @@ -0,0 +1,41 @@ +package main + +import ( + "io" + "mime/multipart" + + "go.uber.org/zap" +) + +type MultipartFile interface { + io.ReadCloser + FileName() string +} + +func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartFile, error) { + reader := multipart.NewReader(r, boundary) + + for { + part, err := reader.NextPart() + if err != nil { + return nil, err + } + + name := part.FormName() + if name == "" { + l.Debug("ignore part, empty form name") + continue + } + + filename := part.FileName() + + // ignore multipart/form-data values + if filename == "" { + l.Debug("ignore part, empty filename", zap.String("form", name)) + + continue + } + + return part, nil + } +} diff --git a/upload.go b/upload.go index 38d53e9..0bf99b2 100644 --- a/upload.go +++ b/upload.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "io" - "mime/multipart" "strconv" "time" @@ -37,10 +36,8 @@ func (pr *putResponse) Encode(w io.Writer) error { func (a *app) upload(c *fasthttp.RequestCtx) { var ( err error - name string - tmp io.Reader + file MultipartFile addr *object.Address - form *multipart.Form cid = container.NewID() sCID, _ = c.UserValue("cid").(string) @@ -55,54 +52,24 @@ func (a *app) upload(c *fasthttp.RequestCtx) { defer func() { // if temporary reader can be closed - close it - if closer, ok := tmp.(io.Closer); ok && closer != nil { - log.Debug("close temporary multipart/form file", zap.Error(closer.Close())) - } - - if form == nil { + if file == nil { return } - log.Debug("cleanup multipart/form", zap.Error(form.RemoveAll())) + log.Debug("close temporary multipart/form file", + zap.Stringer("address", addr), + zap.String("filename", file.FileName()), + zap.Error(file.Close())) }() - // tries to receive multipart/form or throw error - if form, err = c.MultipartForm(); err != nil { + boundary := string(c.Request.Header.MultipartFormBoundary()) + if file, err = fetchMultipartFile(a.log, c.RequestBodyStream(), boundary); err != nil { log.Error("could not receive multipart/form", zap.Error(err)) c.Error("could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } - // checks that received multipart/form contains only one `file` per request - if ln := len(form.File); ln != 1 { - log.Error("received multipart/form with more then one file", zap.Int("count", ln)) - c.Error("received multipart/form with more then one file", fasthttp.StatusBadRequest) - - return - } - - for _, file := range form.File { - // because multipart/form can contains multiple FileHeader records - // we should check that we have only one per request or throw error - if ln := len(file); ln != 1 { - log.Error("received multipart/form file should contains one record", zap.Int("count", ln)) - c.Error("received multipart/form file should contains one record", fasthttp.StatusBadRequest) - - return - } - - name = file[0].Filename - - // opens multipart/form file to work within or throw error - if tmp, err = file[0].Open(); err != nil { - log.Error("could not prepare uploaded file", zap.Error(err)) - c.Error("could not prepare uploaded file", fasthttp.StatusBadRequest) - - return - } - } - filtered := filterHeaders(a.log, &c.Request.Header) attributes := make([]*object.Attribute, 0, len(filtered)) @@ -119,7 +86,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { if _, ok := filtered[object.AttributeFileName]; !ok { filename := object.NewAttribute() filename.SetKey(object.AttributeFileName) - filename.SetValue(name) + filename.SetValue(file.FileName()) attributes = append(attributes, filename) } @@ -140,7 +107,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { raw.SetAttributes(attributes...) // tries to put file into NeoFS or throw error - if addr, err = a.cli.Object().Put(c, raw.Object(), sdk.WithPutReader(tmp)); err != nil { + if addr, err = a.cli.Object().Put(c, raw.Object(), sdk.WithPutReader(file)); err != nil { log.Error("could not store file in NeoFS", zap.Error(err)) c.Error("could not store file in NeoFS", fasthttp.StatusBadRequest) From b8d10122a9dd2aa058339691168b702395a30bcb Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 25 Jan 2021 16:20:08 +0300 Subject: [PATCH 108/548] Add method to fetch BearerToken from Header / Cookies - `BearerToken` allow to parse header/cookies to fetch bearer token - `headerAuth` allow to parse header to fetch bearer token - `cookieAuth` allow to parse header to fetch bearer token closes #2 closes #11 Signed-off-by: Evgeniy Kulikov --- header.go | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 header.go diff --git a/header.go b/header.go new file mode 100644 index 0000000..c359054 --- /dev/null +++ b/header.go @@ -0,0 +1,76 @@ +package main + +import ( + "bytes" + "encoding/base64" + + "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/pkg/errors" + "github.com/valyala/fasthttp" +) + +const bearerToken = "Bearer" + +// BearerToken usage: +// +// if tkn, err = BearerToken(c); err != nil && tkn == nil { +// log.Error("could not fetch bearer token", zap.Error(err)) +// c.Error("could not fetch bearer token", fasthttp.StatusBadRequest) +// return +// } +var _ = BearerToken + +func headerAuth(h *fasthttp.RequestHeader) (*token.BearerToken, error) { + auth := h.Peek(fasthttp.HeaderAuthorization) + if auth == nil || !bytes.Contains(auth, []byte(bearerToken)) { + return nil, nil + } + + auth = bytes.ReplaceAll(auth, []byte(bearerToken+" "), nil) + + data, err := base64.StdEncoding.DecodeString(string(auth)) + if err != nil { + return nil, errors.Wrap(err, "could not fetch marshaled from base64") + } + + tkn := new(token.BearerToken) + if err = tkn.Unmarshal(data); err != nil { + return nil, errors.Wrap(err, "could unmarshal bearer token") + } + + return tkn, nil +} + +func cookieAuth(h *fasthttp.RequestHeader) (*token.BearerToken, error) { + auth := h.Cookie(bearerToken) + if auth == nil { + return nil, nil + } + + data, err := base64.StdEncoding.DecodeString(string(auth)) + if err != nil { + return nil, errors.Wrap(err, "could not fetch marshaled from base64") + } + + tkn := new(token.BearerToken) + if err = tkn.Unmarshal(data); err != nil { + return nil, errors.Wrap(err, "could unmarshal bearer token") + } + + return tkn, nil +} + +func BearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { + // ignore empty value + if ctx == nil { + return nil, nil + } + + if tkn, err := headerAuth(&ctx.Request.Header); err != nil { + return nil, err + } else if tkn != nil { + return tkn, nil + } + + return cookieAuth(&ctx.Request.Header) +} From e45e5ed6f931883b2b964644cce5c654c570bfc9 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 25 Jan 2021 17:26:08 +0300 Subject: [PATCH 109/548] Refactoring and add tests - rename `header.go` to `bearer.go` - simplify header / cookie fetching of bearer token value - simplify `BearerToken` method - increase tests Signed-off-by: Evgeniy Kulikov --- bearer.go | 79 ++++++++++++++++++++++++++++ bearer_test.go | 139 +++++++++++++++++++++++++++++++++++++++++++++++++ header.go | 76 --------------------------- 3 files changed, 218 insertions(+), 76 deletions(-) create mode 100644 bearer.go create mode 100644 bearer_test.go delete mode 100644 header.go diff --git a/bearer.go b/bearer.go new file mode 100644 index 0000000..f98b41d --- /dev/null +++ b/bearer.go @@ -0,0 +1,79 @@ +package main + +import ( + "bytes" + "encoding/base64" + + "github.com/pkg/errors" + + "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/valyala/fasthttp" +) + +type fromHandler = func(h *fasthttp.RequestHeader) []byte + +const bearerToken = "Bearer" + +// BearerToken usage: +// +// if tkn, err = BearerToken(c); err != nil && tkn == nil { +// log.Error("could not fetch bearer token", zap.Error(err)) +// c.Error("could not fetch bearer token", fasthttp.StatusBadRequest) +// return +// } +var _ = BearerToken + +func fromHeader(h *fasthttp.RequestHeader) []byte { + auth := h.Peek(fasthttp.HeaderAuthorization) + if auth == nil || !bytes.HasPrefix(auth, []byte(bearerToken)) { + return nil + } + + if auth = bytes.TrimPrefix(auth, []byte(bearerToken+" ")); len(auth) == 0 { + return nil + } + + return auth +} + +func fromCookie(h *fasthttp.RequestHeader) []byte { + auth := h.Cookie(bearerToken) + if len(auth) == 0 { + return nil + } + + return auth +} + +func BearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { + // ignore empty value + if ctx == nil { + panic(nil) + return nil, nil + } + + var ( + lastErr error + + buf []byte + tkn = new(token.BearerToken) + ) + + for _, parse := range []fromHandler{fromHeader, fromCookie} { + if buf = parse(&ctx.Request.Header); buf == nil { + continue + } else if data, err := base64.StdEncoding.DecodeString(string(buf)); err != nil { + lastErr = errors.Wrap(err, "could not fetch marshaled from base64") + continue + } else if err = tkn.Unmarshal(data); err != nil { + lastErr = errors.Wrap(err, "could not unmarshal bearer token") + continue + } else if tkn == nil { + continue + } + + return tkn, nil + } + + return nil, lastErr +} diff --git a/bearer_test.go b/bearer_test.go new file mode 100644 index 0000000..f9d998d --- /dev/null +++ b/bearer_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "encoding/base64" + "testing" + + "github.com/nspcc-dev/neofs-api-go/pkg/owner" + + "github.com/nspcc-dev/neofs-api-go/pkg/token" + + "github.com/stretchr/testify/require" + "github.com/valyala/fasthttp" +) + +func makeTestCookie(value []byte) *fasthttp.RequestHeader { + header := new(fasthttp.RequestHeader) + header.SetCookie(bearerToken, string(value)) + return header +} + +func makeTestHeader(value []byte) *fasthttp.RequestHeader { + header := new(fasthttp.RequestHeader) + if value != nil { + header.Set(fasthttp.HeaderAuthorization, bearerToken+" "+string(value)) + } + return header +} + +func Test_fromCookie(t *testing.T) { + cases := []struct { + name string + actual []byte + expect []byte + }{ + {name: "empty"}, + {name: "normal", actual: []byte("TOKEN"), expect: []byte("TOKEN")}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expect, fromCookie(makeTestCookie(tt.actual))) + }) + } +} + +func Test_fromHeader(t *testing.T) { + cases := []struct { + name string + actual []byte + expect []byte + }{ + {name: "empty"}, + {name: "normal", actual: []byte("TOKEN"), expect: []byte("TOKEN")}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expect, fromHeader(makeTestHeader(tt.actual))) + }) + } +} + +func TestBearerToken(t *testing.T) { + uid := owner.NewID() + + tkn := new(token.BearerToken) + tkn.SetOwner(uid) + + data, err := tkn.Marshal() + + require.NoError(t, err) + + t64 := base64.StdEncoding.EncodeToString(data) + require.NotEmpty(t, t64) + + cases := []struct { + name string + + cookie string + header string + + error string + expect *token.BearerToken + }{ + {name: "empty"}, + + {name: "bad base64 header", header: "WRONG BASE64", error: "could not fetch marshaled from base64"}, + {name: "bad base64 cookie", cookie: "WRONG BASE64", error: "could not fetch marshaled from base64"}, + + {name: "header token unmarshal error", header: "dGVzdAo=", error: "could not unmarshal bearer token"}, + {name: "cookie token unmarshal error", cookie: "dGVzdAo=", error: "could not unmarshal bearer token"}, + + { + name: "bad header and cookie", + header: "WRONG BASE64", + cookie: "dGVzdAo=", + error: "could not unmarshal bearer token", + }, + + { + name: "bad header, but good cookie", + header: "dGVzdAo=", + cookie: t64, + expect: tkn, + }, + + {name: "ok for header", header: t64, expect: tkn}, + {name: "ok for cookie", cookie: t64, expect: tkn}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + ctx := makeTestRequest(tt.cookie, tt.header) + actual, err := BearerToken(ctx) + + if tt.error == "" { + require.NoError(t, err) + require.Equal(t, tt.expect, actual) + + return + } + + require.Contains(t, err.Error(), tt.error) + }) + } +} + +func makeTestRequest(cookie, header string) *fasthttp.RequestCtx { + ctx := new(fasthttp.RequestCtx) + + if cookie != "" { + ctx.Request.Header.SetCookie(bearerToken, cookie) + } + + if header != "" { + ctx.Request.Header.Set(fasthttp.HeaderAuthorization, bearerToken+" "+header) + } + return ctx +} diff --git a/header.go b/header.go deleted file mode 100644 index c359054..0000000 --- a/header.go +++ /dev/null @@ -1,76 +0,0 @@ -package main - -import ( - "bytes" - "encoding/base64" - - "github.com/nspcc-dev/neofs-api-go/pkg/token" - "github.com/pkg/errors" - "github.com/valyala/fasthttp" -) - -const bearerToken = "Bearer" - -// BearerToken usage: -// -// if tkn, err = BearerToken(c); err != nil && tkn == nil { -// log.Error("could not fetch bearer token", zap.Error(err)) -// c.Error("could not fetch bearer token", fasthttp.StatusBadRequest) -// return -// } -var _ = BearerToken - -func headerAuth(h *fasthttp.RequestHeader) (*token.BearerToken, error) { - auth := h.Peek(fasthttp.HeaderAuthorization) - if auth == nil || !bytes.Contains(auth, []byte(bearerToken)) { - return nil, nil - } - - auth = bytes.ReplaceAll(auth, []byte(bearerToken+" "), nil) - - data, err := base64.StdEncoding.DecodeString(string(auth)) - if err != nil { - return nil, errors.Wrap(err, "could not fetch marshaled from base64") - } - - tkn := new(token.BearerToken) - if err = tkn.Unmarshal(data); err != nil { - return nil, errors.Wrap(err, "could unmarshal bearer token") - } - - return tkn, nil -} - -func cookieAuth(h *fasthttp.RequestHeader) (*token.BearerToken, error) { - auth := h.Cookie(bearerToken) - if auth == nil { - return nil, nil - } - - data, err := base64.StdEncoding.DecodeString(string(auth)) - if err != nil { - return nil, errors.Wrap(err, "could not fetch marshaled from base64") - } - - tkn := new(token.BearerToken) - if err = tkn.Unmarshal(data); err != nil { - return nil, errors.Wrap(err, "could unmarshal bearer token") - } - - return tkn, nil -} - -func BearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { - // ignore empty value - if ctx == nil { - return nil, nil - } - - if tkn, err := headerAuth(&ctx.Request.Header); err != nil { - return nil, err - } else if tkn != nil { - return tkn, nil - } - - return cookieAuth(&ctx.Request.Header) -} From 237c247ec4c2d929d0041343c8767f4ace7efe43 Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Tue, 16 Feb 2021 18:20:15 +0300 Subject: [PATCH 110/548] [#13] Add bearer token usage in receive/upload methods --- bearer.go | 32 ++++++++++++++++++++++---------- bearer_test.go | 38 ++++++++++++++++++++++++++++++-------- go.mod | 1 + receive.go | 6 ++++++ upload.go | 6 ++++++ 5 files changed, 65 insertions(+), 18 deletions(-) diff --git a/bearer.go b/bearer.go index f98b41d..ee01fee 100644 --- a/bearer.go +++ b/bearer.go @@ -4,32 +4,33 @@ import ( "bytes" "encoding/base64" - "github.com/pkg/errors" - "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/pkg/errors" "github.com/valyala/fasthttp" ) type fromHandler = func(h *fasthttp.RequestHeader) []byte -const bearerToken = "Bearer" +const ( + bearerTokenHdr = "Bearer" + bearerTokenKey = "__context_bearer_token_key" +) // BearerToken usage: // -// if tkn, err = BearerToken(c); err != nil && tkn == nil { +// if err = checkAndPropagateBearerToken(ctx); err != nil { // log.Error("could not fetch bearer token", zap.Error(err)) // c.Error("could not fetch bearer token", fasthttp.StatusBadRequest) // return // } -var _ = BearerToken func fromHeader(h *fasthttp.RequestHeader) []byte { auth := h.Peek(fasthttp.HeaderAuthorization) - if auth == nil || !bytes.HasPrefix(auth, []byte(bearerToken)) { + if auth == nil || !bytes.HasPrefix(auth, []byte(bearerTokenHdr)) { return nil } - if auth = bytes.TrimPrefix(auth, []byte(bearerToken+" ")); len(auth) == 0 { + if auth = bytes.TrimPrefix(auth, []byte(bearerTokenHdr+" ")); len(auth) == 0 { return nil } @@ -37,7 +38,7 @@ func fromHeader(h *fasthttp.RequestHeader) []byte { } func fromCookie(h *fasthttp.RequestHeader) []byte { - auth := h.Cookie(bearerToken) + auth := h.Cookie(bearerTokenHdr) if len(auth) == 0 { return nil } @@ -45,10 +46,21 @@ func fromCookie(h *fasthttp.RequestHeader) []byte { return auth } -func BearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { +func checkAndPropagateBearerToken(ctx *fasthttp.RequestCtx) error { + tkn, err := fetchBearerToken(ctx) + if err != nil { + return err + } + + // This is an analog of context.WithValue. + ctx.SetUserValue(bearerTokenKey, tkn) + + return nil +} + +func fetchBearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { // ignore empty value if ctx == nil { - panic(nil) return nil, nil } diff --git a/bearer_test.go b/bearer_test.go index f9d998d..4ae7b20 100644 --- a/bearer_test.go +++ b/bearer_test.go @@ -4,24 +4,23 @@ import ( "encoding/base64" "testing" + sdk "github.com/nspcc-dev/cdn-sdk" "github.com/nspcc-dev/neofs-api-go/pkg/owner" - "github.com/nspcc-dev/neofs-api-go/pkg/token" - "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" ) func makeTestCookie(value []byte) *fasthttp.RequestHeader { header := new(fasthttp.RequestHeader) - header.SetCookie(bearerToken, string(value)) + header.SetCookie(bearerTokenHdr, string(value)) return header } func makeTestHeader(value []byte) *fasthttp.RequestHeader { header := new(fasthttp.RequestHeader) if value != nil { - header.Set(fasthttp.HeaderAuthorization, bearerToken+" "+string(value)) + header.Set(fasthttp.HeaderAuthorization, bearerTokenHdr+" "+string(value)) } return header } @@ -60,7 +59,7 @@ func Test_fromHeader(t *testing.T) { } } -func TestBearerToken(t *testing.T) { +func Test_fetchBearerToken(t *testing.T) { uid := owner.NewID() tkn := new(token.BearerToken) @@ -111,7 +110,7 @@ func TestBearerToken(t *testing.T) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { ctx := makeTestRequest(tt.cookie, tt.header) - actual, err := BearerToken(ctx) + actual, err := fetchBearerToken(ctx) if tt.error == "" { require.NoError(t, err) @@ -129,11 +128,34 @@ func makeTestRequest(cookie, header string) *fasthttp.RequestCtx { ctx := new(fasthttp.RequestCtx) if cookie != "" { - ctx.Request.Header.SetCookie(bearerToken, cookie) + ctx.Request.Header.SetCookie(bearerTokenHdr, cookie) } if header != "" { - ctx.Request.Header.Set(fasthttp.HeaderAuthorization, bearerToken+" "+header) + ctx.Request.Header.Set(fasthttp.HeaderAuthorization, bearerTokenHdr+" "+header) } return ctx } + +func Test_checkAndPropagateBearerToken(t *testing.T) { + uid := owner.NewID() + + tkn := new(token.BearerToken) + tkn.SetOwner(uid) + + data, err := tkn.Marshal() + require.NoError(t, err) + + t64 := base64.StdEncoding.EncodeToString(data) + require.NotEmpty(t, t64) + + ctx := makeTestRequest(t64, "") + + // Expect to see the token within the context. + require.NoError(t, checkAndPropagateBearerToken(ctx)) + + // Expect to see the same token without errors. + actual, err := sdk.BearerToken(ctx) + require.NoError(t, err) + require.Equal(t, tkn, actual) +} diff --git a/go.mod b/go.mod index f8e4e05..6e7828b 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 + github.com/stretchr/testify v1.7.0 github.com/valyala/fasthttp v1.20.0 go.uber.org/zap v1.16.0 google.golang.org/grpc v1.35.0 diff --git a/receive.go b/receive.go index 5139252..ae1b555 100644 --- a/receive.go +++ b/receive.go @@ -57,6 +57,12 @@ func (r *request) receiveFile(address *object.Address) { filename string ) + if err = checkAndPropagateBearerToken(r.RequestCtx); err != nil { + r.log.Error("could not fetch bearer token", zap.Error(err)) + r.Error("could not fetch bearer token", fasthttp.StatusBadRequest) + return + } + writer := newDetector(r.Response.BodyWriter()) obj, err := r.obj.Get(r, address, sdk.WithGetWriter(writer)) if err != nil { diff --git a/upload.go b/upload.go index 0bf99b2..3124e26 100644 --- a/upload.go +++ b/upload.go @@ -44,6 +44,12 @@ func (a *app) upload(c *fasthttp.RequestCtx) { log = a.log.With(zap.String("cid", sCID)) ) + if err = checkAndPropagateBearerToken(c); err != nil { + log.Error("could not fetch bearer token", zap.Error(err)) + c.Error("could not fetch bearer token", fasthttp.StatusBadRequest) + return + } + if err = cid.Parse(sCID); err != nil { log.Error("wrong container id", zap.Error(err)) c.Error("wrong container id", fasthttp.StatusBadRequest) From f77c4e49dc7a0037e28904c7c31eac21db4df628 Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Tue, 16 Feb 2021 18:38:30 +0300 Subject: [PATCH 111/548] [#18] Add fetching owner variously on file uploading Signed-off-by: Pavel Korotkov --- upload.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/upload.go b/upload.go index 3124e26..1360ec5 100644 --- a/upload.go +++ b/upload.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "io" "strconv" @@ -9,6 +10,7 @@ import ( sdk "github.com/nspcc-dev/cdn-sdk" "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -33,6 +35,14 @@ func (pr *putResponse) Encode(w io.Writer) error { return enc.Encode(pr) } +func (a *app) fetchOwner(ctx context.Context) *owner.ID { + if tkn, err := sdk.BearerToken(ctx); err == nil && tkn != nil { + return tkn.Issuer() + } + + return a.cli.Owner() +} + func (a *app) upload(c *fasthttp.RequestCtx) { var ( err error @@ -109,7 +119,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { // prepares new object and fill it raw := object.NewRaw() raw.SetContainerID(cid) - raw.SetOwnerID(a.cli.Owner()) // should be various: from sdk / BearerToken + raw.SetOwnerID(a.fetchOwner(c)) raw.SetAttributes(attributes...) // tries to put file into NeoFS or throw error From cdab794d6218fdfee354fbd029e188a9f3396edb Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Thu, 25 Mar 2021 11:27:59 +0300 Subject: [PATCH 112/548] Remove sdk context dep Signed-off-by: Pavel Korotkov --- global/context.go | 20 ++++++++++++++++++++ main.go | 11 +++-------- 2 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 global/context.go diff --git a/global/context.go b/global/context.go new file mode 100644 index 0000000..6aa6754 --- /dev/null +++ b/global/context.go @@ -0,0 +1,20 @@ +package global + +import ( + "context" + "os/signal" + "sync" + "syscall" +) + +var ( + globalContext context.Context + globalContexOnce sync.Once +) + +func Context() context.Context { + globalContexOnce.Do(func() { + globalContext, _ = signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + }) + return globalContext +} diff --git a/main.go b/main.go index 770d5a8..3fb5c6c 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,8 @@ package main import ( - "github.com/nspcc-dev/cdn-sdk/grace" "github.com/nspcc-dev/cdn-sdk/logger" + "github.com/nspcc-dev/neofs-http-gate/global" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -41,15 +41,10 @@ func main() { var ( v = settings() l = newLogger(v) - g = grace.Context(l) - - a = newApp(g, - WithLogger(l), - WithConfig(v)) + g = global.Context() + a = newApp(g, WithLogger(l), WithConfig(v)) ) - go a.Serve(g) go a.Worker(g) - a.Wait() } From 4c96885a42dbcf1f43f4cdde0b3105a68d7af11a Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Wed, 31 Mar 2021 01:46:33 +0300 Subject: [PATCH 113/548] [#19] Add a version with no cdn-sdk deps Signed-off-by: Pavel Korotkov --- .test.env | 10 +- README.md | 60 ++++----- app.go | 138 ++++++++------------ bearer.go | 16 ++- bearer_test.go | 5 +- global/context.go | 9 +- go.mod | 13 +- go.sum | 88 ++++++++++++- go_dev.mod | 2 +- health.go | 2 - logger/grpc.go | 60 +++++++++ logger/option.go | 23 ++++ logger/zap.go | 131 +++++++++++++++++++ main.go | 65 +++++----- neofs/client-plant.go | 286 ++++++++++++++++++++++++++++++++++++++++++ neofs/credentials.go | 67 ++++++++++ pool/pool.go | 39 ++++++ receive.go | 101 ++++++++------- settings.go | 12 +- upload.go | 69 +++++----- 20 files changed, 930 insertions(+), 266 deletions(-) create mode 100644 logger/grpc.go create mode 100644 logger/option.go create mode 100644 logger/zap.go create mode 100644 neofs/client-plant.go create mode 100644 neofs/credentials.go create mode 100644 pool/pool.go diff --git a/.test.env b/.test.env index 569bfda..995d4a3 100644 --- a/.test.env +++ b/.test.env @@ -1,5 +1,5 @@ HTTP_GW_VERBOSE=--true -HTTP_GW_KEY=generated +HTTP_GW_KEY=/home/pk/devbox/work/neo/user.key HTTP_GW_LISTEN_ADDRESS=0.0.0.0:8087 HTTP_GW_LOGGER_LEVEL=debug @@ -10,10 +10,4 @@ HTTP_GW_KEEPALIVE_TIME=120s HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=True HTTP_GW_CONN_TTL=1h HTTP_GW_PEERS_0_WEIGHT=1.0 -HTTP_GW_PEERS_1_WEIGHT=0.125 -HTTP_GW_PEERS_2_WEIGHT=0.125 -HTTP_GW_PEERS_3_WEIGHT=0.125 -HTTP_GW_PEERS_4_WEIGHT=0.125 -HTTP_GW_PEERS_5_WEIGHT=0.125 -HTTP_GW_PEERS_6_WEIGHT=0.125 -HTTP_GW_PEERS_7_WEIGHT=0.125 \ No newline at end of file +HTTP_GW_PEERS_0_ADDRESS=s01.neofs.devenv:8080 \ No newline at end of file diff --git a/README.md b/README.md index 73aaeda..c28063a 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,8 @@ NeoFS HTTP Gate is example of tool that provides basic interactions with NeoFS. ## Configuration ``` -# Flags +# Flags: + --pprof enable pprof --metrics enable prometheus -h, --help show help @@ -43,35 +44,36 @@ NeoFS HTTP Gate is example of tool that provides basic interactions with NeoFS. --verbose debug gRPC connections --request_timeout duration gRPC request timeout (default 5s) --connect_timeout duration gRPC connect timeout (default 30s) - --listen_address string HTTP Gate listen address (default "0.0.0.0:8082") + --listen_address string HTTP gate's listen address (default "0.0.0.0:8082") -p, --peers stringArray NeoFS nodes # Environments: -HTTP_GW_KEY=string - "generated" to generate key, path to private key file, hex string or wif (default "generated") -HTTP_GW_CONNECT_TIMEOUT=Duration - timeout for connection -HTTP_GW_REQUEST_TIMEOUT=Duration - timeout for request -HTTP_GW_REBALANCE_TIMER=Duration - time between connections checks -HTTP_GW_LISTEN_ADDRESS=host:port - address to listen connections -HTTP_GW_PEERS__ADDRESS=host:port - address of NeoFS Node -HTTP_GW_PEERS__WEIGHT=float - weight of NeoFS Node -HTTP_GW_PPROF=bool - enable/disable pprof (/debug/pprof) -HTTP_GW_METRICS=bool - enable/disable prometheus metrics endpoint (/metrics) -HTTP_GW_LOGGER_FORMAT=string - logger format -HTTP_GW_LOGGER_LEVEL=string - logger level -HTTP_GW_LOGGER_NO_CALLER=bool - logger don't show caller -HTTP_GW_LOGGER_NO_DISCLAIMER=bool - logger don't show application name/version -HTTP_GW_LOGGER_SAMPLING_INITIAL=int - logger sampling initial -HTTP_GW_LOGGER_SAMPLING_THEREAFTER=int - logger sampling thereafter -HTTP_GW_LOGGER_TRACE_LEVEL=string - logger show trace on level -HTTP_GW_KEEPALIVE_TIME=Duration - аfter a duration of this time if the client doesn't see any activity -it pings the server to see if the transport is still alive. -HTTP_GW_KEEPALIVE_TIMEOUT=Duration - after having pinged for keepalive check, the client waits for a duration -of Timeout and if no activity is seen even after that the connection is closed -HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=Bool - if true, client sends keepalive pings even with no active RPCs. -If false, when there are no active RPCs, Time and Timeout will be ignored and no keepalive pings will be sent. - -HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=bool - enable/disable adding current timestamp attribute when object uploads +HTTP_GW_KEY=string - Path to private key file, hex string or wif string +HTTP_GW_CONNECT_TIMEOUT=duration - Timeout for connection +HTTP_GW_REQUEST_TIMEOUT=duration - Timeout for request +HTTP_GW_REBALANCE_TIMER=duration - Time between connections checks +HTTP_GW_LISTEN_ADDRESS=host:port - Address to listen connections +HTTP_GW_PEERS__ADDRESS=host:port - Address of NeoFS Node +HTTP_GW_PEERS__WEIGHT=float - Weight of NeoFS Node +HTTP_GW_PPROF=bool - Enable/disable pprof (/debug/pprof) +HTTP_GW_METRICS=bool - Enable/disable prometheus metrics endpoint (/metrics) +HTTP_GW_LOGGER_FORMAT=string - Logger format +HTTP_GW_LOGGER_LEVEL=string - Logger level +HTTP_GW_LOGGER_NO_CALLER=bool - Logger don't show caller +HTTP_GW_LOGGER_NO_DISCLAIMER=bool - Logger don't show application name/version +HTTP_GW_LOGGER_SAMPLING_INITIAL=int - Logger sampling initial +HTTP_GW_LOGGER_SAMPLING_THEREAFTER=int - Logger sampling thereafter +HTTP_GW_LOGGER_TRACE_LEVEL=string - Logger show trace on level +HTTP_GW_KEEPALIVE_TIME=duration - After a duration of this time if the client sees no activity + it pings the server to see if the transport is still alive +HTTP_GW_KEEPALIVE_TIMEOUT=duration - After having pinged for keepalive check, the client waits for a duration + of Timeout and if no activity is seen even after that the connection + is closed +HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=bool - If true, client sends keepalive pings even with no active RPCs. + If false, when there are no active RPCs, Time and Timeout will be + ignored and no keepalive pings will be sent +HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=bool - Enable/disable adding current timestamp attribute when object uploads HTTP_GW_WEB_READ_BUFFER_SIZE=4096 - per-connection buffer size for requests' reading HTTP_GW_WEB_READ_TIMEOUT=15s - an amount of time allowed to read the full request including body @@ -84,6 +86,6 @@ HTTP_GW_WEB_MAX_REQUEST_BODY_SIZE=4194304 - maximum request body size, server r Peers preset: -HTTP_GW_PEERS_[N]_ADDRESS = string -HTTP_GW_PEERS_[N]_WEIGHT = 0..1 (float) -``` \ No newline at end of file +HTTP_GW_PEERS__ADDRESS = string +HTTP_GW_PEERS__WEIGHT = 0..1 (float) +``` diff --git a/app.go b/app.go index 6c70b90..8fe8960 100644 --- a/app.go +++ b/app.go @@ -2,36 +2,33 @@ package main import ( "context" - "crypto/ecdsa" + "sort" "strconv" "github.com/fasthttp/router" - sdk "github.com/nspcc-dev/cdn-sdk" - "github.com/nspcc-dev/cdn-sdk/creds/neofs" - "github.com/nspcc-dev/cdn-sdk/logger" - "github.com/nspcc-dev/cdn-sdk/pool" + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-http-gate/logger" + "github.com/nspcc-dev/neofs-http-gate/neofs" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" - "google.golang.org/grpc" "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/keepalive" ) type ( app struct { - cli sdk.Client - pool pool.Client - log *zap.Logger - cfg *viper.Viper - key *ecdsa.PrivateKey - - wlog logger.Logger - web *fasthttp.Server - - jobDone chan struct{} - webDone chan struct{} - + plant neofs.ClientPlant + getOperations struct { + client client.Client + sessionToken *token.SessionToken + } + log *zap.Logger + cfg *viper.Viper + wlog logger.Logger + web *fasthttp.Server + jobDone chan struct{} + webDone chan struct{} enableDefaultTimestamp bool } @@ -84,11 +81,11 @@ func newApp(ctx context.Context, opt ...Option) App { grpclog.SetLoggerV2(a.wlog) } - conTimeout := a.cfg.GetDuration(cfgConTimeout) - reqTimeout := a.cfg.GetDuration(cfgReqTimeout) - tckTimeout := a.cfg.GetDuration(cfgRebalance) + // conTimeout := a.cfg.GetDuration(cfgConTimeout) + // reqTimeout := a.cfg.GetDuration(cfgReqTimeout) + // tckTimeout := a.cfg.GetDuration(cfgRebalance) - // -- setup FastHTTP server: -- + // -- setup FastHTTP server -- a.web.Name = "neofs-http-gate" a.web.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) a.web.WriteBufferSize = a.cfg.GetInt(cfgWebWriteBufferSize) @@ -99,68 +96,42 @@ func newApp(ctx context.Context, opt ...Option) App { a.web.NoDefaultContentType = true a.web.MaxRequestBodySize = a.cfg.GetInt(cfgWebMaxRequestBodySize) - // FIXME don't work with StreamRequestBody, - // some bugs with readMultipartForm + // -- -- -- -- -- -- FIXME -- -- -- -- -- -- + // Does not work with StreamRequestBody, + // some bugs with readMultipartForm // https://github.com/valyala/fasthttp/issues/968 a.web.DisablePreParseMultipartForm = true - a.web.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) - // -- -- -- -- -- -- -- -- -- -- + // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - connections := make(map[string]float64) + var cl connectionList for i := 0; ; i++ { address := a.cfg.GetString(cfgPeers + "." + strconv.Itoa(i) + ".address") weight := a.cfg.GetFloat64(cfgPeers + "." + strconv.Itoa(i) + ".weight") if address == "" { break } - - connections[address] = weight - a.log.Info("add connection peer", - zap.String("address", address), - zap.Float64("weight", weight)) + cl = append(cl, connection{address: address, weight: weight}) + a.log.Info("add connection peer", zap.String("address", address), zap.Float64("weight", weight)) } - - cred, err := neofs.New(a.cfg.GetString(cmdNeoFSKey)) + sort.Sort(sort.Reverse(cl)) + cred, err := neofs.NewCredentials(a.cfg.GetString(cmdNeoFSKey)) if err != nil { - a.log.Fatal("could not prepare credentials", zap.Error(err)) + a.log.Fatal("could not get credentials", zap.Error(err)) } - - a.pool, err = pool.New(ctx, - pool.WithLogger(a.log), - pool.WithCredentials(cred), - pool.WithWeightPool(connections), - pool.WithTickerTimeout(tckTimeout), - pool.WithConnectTimeout(conTimeout), - pool.WithRequestTimeout(reqTimeout), - pool.WithAPIPreparer(sdk.APIPreparer), - pool.WithGRPCOptions( - grpc.WithBlock(), - grpc.WithInsecure(), - grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: a.cfg.GetDuration(cfgKeepaliveTime), - Timeout: a.cfg.GetDuration(cfgKeepaliveTimeout), - PermitWithoutStream: a.cfg.GetBool(cfgKeepalivePermitWithoutStream), - }))) - + a.plant, err = neofs.NewClientPlant(ctx, cl[0].address, cred) if err != nil { - a.log.Fatal("could not prepare connection pool", zap.Error(err)) + a.log.Fatal("failed to create neofs client") } - - a.cli, err = sdk.New(ctx, - sdk.WithLogger(a.log), - sdk.WithCredentials(cred), - sdk.WithConnectionPool(a.pool), - sdk.WithAPIPreparer(sdk.APIPreparer)) + a.getOperations.client, a.getOperations.sessionToken, err = a.plant.GetReusableArtifacts(ctx) if err != nil { - a.log.Fatal("could not prepare sdk client", zap.Error(err)) + a.log.Fatal("failed to get neofs client's reusable artifacts") } - return a } func (a *app) Wait() { - a.log.Info("application started") + a.log.Info("starting application") select { case <-a.jobDone: // wait for job is stopped @@ -171,50 +142,51 @@ func (a *app) Wait() { } func (a *app) Worker(ctx context.Context) { - a.pool.Worker(ctx) close(a.jobDone) } func (a *app) Serve(ctx context.Context) { go func() { <-ctx.Done() - a.log.Info("stop web-server", zap.Error(a.web.Shutdown())) + a.log.Info("shutting down web server", zap.Error(a.web.Shutdown())) close(a.webDone) }() - + // Configure router. r := router.New() r.RedirectTrailingSlash = true - - a.log.Info("enabled /upload/{cid}") r.POST("/upload/{cid}", a.upload) - - a.log.Info("enabled /get/{cid}/{oid}") + a.log.Info("added path /upload/{cid}") r.GET("/get/{cid}/{oid}", a.byAddress) - - a.log.Info("enabled /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") + a.log.Info("added path /get/{cid}/{oid}") r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.byAttribute) - + a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") // attaching /-/(ready,healthy) - attachHealthy(r, a.pool.Status) - + // attachHealthy(r, a.pool.Status) // enable metrics if a.cfg.GetBool(cmdMetrics) { - a.log.Info("enabled /metrics/") + a.log.Info("added path /metrics/") attachMetrics(r, a.wlog) } - // enable pprof if a.cfg.GetBool(cmdPprof) { - a.log.Info("enabled /debug/pprof/") + a.log.Info("added path /debug/pprof/") attachProfiler(r) } - bind := a.cfg.GetString(cfgListenAddress) - a.log.Info("run gateway server", - zap.String("address", bind)) - + a.log.Info("running web server", zap.String("address", bind)) a.web.Handler = r.Handler if err := a.web.ListenAndServe(bind); err != nil { a.log.Fatal("could not start server", zap.Error(err)) } } + +type connection struct { + address string + weight float64 +} + +type connectionList []connection + +func (p connectionList) Len() int { return len(p) } +func (p connectionList) Less(i, j int) bool { return p[i].weight < p[j].weight } +func (p connectionList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/bearer.go b/bearer.go index ee01fee..e14c35c 100644 --- a/bearer.go +++ b/bearer.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "encoding/base64" "github.com/nspcc-dev/neofs-api-go/pkg/token" @@ -18,7 +19,7 @@ const ( // BearerToken usage: // -// if err = checkAndPropagateBearerToken(ctx); err != nil { +// if err = storeBearerToken(ctx); err != nil { // log.Error("could not fetch bearer token", zap.Error(err)) // c.Error("could not fetch bearer token", fasthttp.StatusBadRequest) // return @@ -46,15 +47,13 @@ func fromCookie(h *fasthttp.RequestHeader) []byte { return auth } -func checkAndPropagateBearerToken(ctx *fasthttp.RequestCtx) error { +func storeBearerToken(ctx *fasthttp.RequestCtx) error { tkn, err := fetchBearerToken(ctx) if err != nil { return err } - // This is an analog of context.WithValue. ctx.SetUserValue(bearerTokenKey, tkn) - return nil } @@ -63,14 +62,12 @@ func fetchBearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { if ctx == nil { return nil, nil } - var ( lastErr error buf []byte tkn = new(token.BearerToken) ) - for _, parse := range []fromHandler{fromHeader, fromCookie} { if buf = parse(&ctx.Request.Header); buf == nil { continue @@ -89,3 +86,10 @@ func fetchBearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { return nil, lastErr } + +func loadBearerToken(ctx context.Context) (*token.BearerToken, error) { + if tkn, ok := ctx.Value(bearerTokenKey).(*token.BearerToken); ok && tkn != nil { + return tkn, nil + } + return nil, errors.New("found empty bearer token") +} diff --git a/bearer_test.go b/bearer_test.go index 4ae7b20..4cfa31f 100644 --- a/bearer_test.go +++ b/bearer_test.go @@ -4,7 +4,6 @@ import ( "encoding/base64" "testing" - sdk "github.com/nspcc-dev/cdn-sdk" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/stretchr/testify/require" @@ -152,10 +151,10 @@ func Test_checkAndPropagateBearerToken(t *testing.T) { ctx := makeTestRequest(t64, "") // Expect to see the token within the context. - require.NoError(t, checkAndPropagateBearerToken(ctx)) + require.NoError(t, storeBearerToken(ctx)) // Expect to see the same token without errors. - actual, err := sdk.BearerToken(ctx) + actual, err := loadBearerToken(ctx) require.NoError(t, err) require.Equal(t, tkn, actual) } diff --git a/global/context.go b/global/context.go index 6aa6754..19b3c53 100644 --- a/global/context.go +++ b/global/context.go @@ -8,13 +8,16 @@ import ( ) var ( - globalContext context.Context - globalContexOnce sync.Once + globalContext context.Context + globalContextOnce sync.Once + globalContextBarrier = make(chan struct{}) ) func Context() context.Context { - globalContexOnce.Do(func() { + globalContextOnce.Do(func() { globalContext, _ = signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + close(globalContextBarrier) }) + <-globalContextBarrier return globalContext } diff --git a/go.mod b/go.mod index 6e7828b..016df60 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,21 @@ module github.com/nspcc-dev/neofs-http-gate -go 1.15 +go 1.16 require ( github.com/fasthttp/router v1.3.5 - github.com/nspcc-dev/cdn-sdk v0.3.4 - github.com/nspcc-dev/neofs-api-go v1.23.0 + github.com/nspcc-dev/neofs-api-go v1.25.0 + github.com/nspcc-dev/neofs-crypto v0.3.0 + github.com/nspcc-dev/neofs-node v0.17.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 - github.com/valyala/fasthttp v1.20.0 + github.com/valyala/fasthttp v1.22.0 + go.uber.org/atomic v1.7.0 // indirect go.uber.org/zap v1.16.0 - google.golang.org/grpc v1.35.0 + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect + google.golang.org/grpc v1.36.1 ) diff --git a/go.sum b/go.sum index b4818b2..439631f 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,7 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -21,6 +22,9 @@ github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1 github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= +github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= +github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -29,6 +33,8 @@ github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzg github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/alecthomas/participle v0.6.0/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY= +github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -38,6 +44,8 @@ github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGn github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -92,6 +100,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -140,6 +149,7 @@ github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFG github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -190,6 +200,7 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -214,6 +225,8 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -232,6 +245,7 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -239,10 +253,14 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo= +github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -270,6 +288,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -285,6 +305,14 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= +github.com/multiformats/go-multiaddr-net v0.1.2/go.mod h1:QsWt3XK/3hwvNxZJp92iMQKME1qHfpYmyIjFVsSOY6Y= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= @@ -294,27 +322,37 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nspcc-dev/cdn-sdk v0.3.4 h1:RtYWuF9xDWrkVwu6sFRWlyZ+ToYM7Y9h8B93Fg4CPEA= -github.com/nspcc-dev/cdn-sdk v0.3.4/go.mod h1:JC4dT16H5HilyZcb8sTxL/TMC1FSEKMuFAqRsmAPoAk= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E= +github.com/nspcc-dev/dbft v0.0.0-20210302103605-cc75991b7cfb/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= +github.com/nspcc-dev/neo-go v0.94.0 h1:2eafoyEnueqEMGDZF06HZJQN0MHgYxFzlcre5YUOP1M= +github.com/nspcc-dev/neo-go v0.94.0/go.mod h1:IrBT/UG3/Slhqgjge8r6zni5tNRsWmwwG9OnVdASheI= github.com/nspcc-dev/neofs-api-go v1.23.0 h1:t4FB5uVY99UkYR0Hiyi1SHjZuqzf4qicw7tf7BBnkHk= github.com/nspcc-dev/neofs-api-go v1.23.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-api-go v1.25.0 h1:hw1TTi3/3wCBB3KN6cycuM8Sn/MFnapSQptBx8KgDIY= +github.com/nspcc-dev/neofs-api-go v1.25.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= +github.com/nspcc-dev/neofs-http-gate/sdk v0.3.4 h1:RtYWuF9xDWrkVwu6sFRWlyZ+ToYM7Y9h8B93Fg4CPEA= +github.com/nspcc-dev/neofs-http-gate/sdk v0.3.4/go.mod h1:JC4dT16H5HilyZcb8sTxL/TMC1FSEKMuFAqRsmAPoAk= +github.com/nspcc-dev/neofs-node v0.17.0 h1:Xh01n8AKC1XmSRBN7yBKSO7Vz2rchqZfAnofcq1lis0= +github.com/nspcc-dev/neofs-node v0.17.0/go.mod h1:nC4+0Zqxv7KZMOv59g5TvcCg0kV6m+1ySiDEuzlAuLc= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= +github.com/nspcc-dev/tzhash v1.4.0 h1:RVIR+mxOBHl58CE99+DXtE31ylD5PEkZSoWqoj4fVjg= +github.com/nspcc-dev/tzhash v1.4.0/go.mod h1:Z8gp/VZbyWgPhaMp/KTmeoW5UTynp/N60g0jTtSzBws= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -335,7 +373,10 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/panjf2000/ants/v2 v2.3.0/go.mod h1:LtwNaBX6OeF5qRtQlaeGndalVwJlS2ueur7uwoAHbPA= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/paulmach/orb v0.2.1/go.mod h1:91bG5A8qKNOiZtlKc0BqKMB3O5kWfRQorTwo8BZ2B/0= +github.com/paulmach/protoscan v0.2.0/go.mod h1:2c55sl1Hu6/tgRfc8Y8zADsxuSCYC2IrPh0JCqP/yrw= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -357,6 +398,7 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU= github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= @@ -372,6 +414,7 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= @@ -382,6 +425,7 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -415,6 +459,7 @@ github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -422,6 +467,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -442,6 +489,7 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -451,6 +499,8 @@ github.com/valyala/fasthttp v1.19.0 h1:PfTS4PeH3xDr3WomrDS2ID8lU2GskK1xS3YG6gIpi github.com/valyala/fasthttp v1.19.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= github.com/valyala/fasthttp v1.20.0 h1:olTmcnLQeZrkBc4TVgE/BatTo1NE/IvW050AuD8SW+U= github.com/valyala/fasthttp v1.20.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= +github.com/valyala/fasthttp v1.22.0 h1:OpwH5KDOJ9cS2bq8fD+KfT4IrksK0llvkHf4MZx42jQ= +github.com/valyala/fasthttp v1.22.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -459,6 +509,7 @@ github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBU go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -468,12 +519,14 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= @@ -489,11 +542,17 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -509,6 +568,7 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -528,15 +588,21 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191105084925-a882066a44e0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226101413-39120d07d75e h1:jIQURUJ9mlLvYwTBtRHm9h58rYhSonLvRvgAnP8Nr7I= +golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -571,23 +637,34 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -596,6 +673,7 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -614,6 +692,7 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200123022218-593de606220b h1:ztSlcncMErSAUzXwnVO1iTPxHwtvOHBB26SGiyYXIEE= golang.org/x/tools v0.0.0-20200123022218-593de606220b/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -660,6 +739,10 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1 h1:cmUfbeGKnz9+2DD/UYsMQXeqbHZqZDs4eQwW0sFOpBY= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -692,6 +775,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= diff --git a/go_dev.mod b/go_dev.mod index 8077c1d..f45f6ae 100644 --- a/go_dev.mod +++ b/go_dev.mod @@ -1,6 +1,6 @@ module github.com/nspcc-dev/neofs-http-gate -go 1.13 +go 1.16 require ( github.com/fasthttp/router v0.6.1 diff --git a/health.go b/health.go index 4483e70..719667b 100644 --- a/health.go +++ b/health.go @@ -17,7 +17,6 @@ func attachHealthy(r *router.Router, e stater) { ctx.SetStatusCode(fasthttp.StatusOK) ctx.SetBodyString(healthyState + "ready") }) - r.GET("/-/healthy/", func(c *fasthttp.RequestCtx) { code := fasthttp.StatusOK msg := "healthy" @@ -26,7 +25,6 @@ func attachHealthy(r *router.Router, e stater) { msg = "unhealthy: " + err.Error() code = fasthttp.StatusBadRequest } - c.Response.Reset() c.SetStatusCode(code) c.SetContentType(defaultContentType) diff --git a/logger/grpc.go b/logger/grpc.go new file mode 100644 index 0000000..b163e7c --- /dev/null +++ b/logger/grpc.go @@ -0,0 +1,60 @@ +package logger + +import ( + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "google.golang.org/grpc/grpclog" +) + +type ( + zapLogger struct { + zapcore.Core + log *zap.SugaredLogger + } + + Logger interface { + grpclog.LoggerV2 + Println(v ...interface{}) + } +) + +func GRPC(l *zap.Logger) Logger { + log := l.WithOptions( + // skip gRPCLog + zapLogger in caller + zap.AddCallerSkip(2)) + + return &zapLogger{ + Core: log.Core(), + log: log.Sugar(), + } +} + +func (z *zapLogger) Info(args ...interface{}) { z.log.Info(args...) } + +func (z *zapLogger) Infoln(args ...interface{}) { z.log.Info(args...) } + +func (z *zapLogger) Infof(format string, args ...interface{}) { z.log.Infof(format, args...) } + +func (z *zapLogger) Println(args ...interface{}) { z.log.Info(args...) } + +func (z *zapLogger) Printf(format string, args ...interface{}) { z.log.Infof(format, args...) } + +func (z *zapLogger) Warning(args ...interface{}) { z.log.Warn(args...) } + +func (z *zapLogger) Warningln(args ...interface{}) { z.log.Warn(args...) } + +func (z *zapLogger) Warningf(format string, args ...interface{}) { z.log.Warnf(format, args...) } + +func (z *zapLogger) Error(args ...interface{}) { z.log.Error(args...) } + +func (z *zapLogger) Errorln(args ...interface{}) { z.log.Error(args...) } + +func (z *zapLogger) Errorf(format string, args ...interface{}) { z.log.Errorf(format, args...) } + +func (z *zapLogger) Fatal(args ...interface{}) { z.log.Fatal(args...) } + +func (z *zapLogger) Fatalln(args ...interface{}) { z.log.Fatal(args...) } + +func (z *zapLogger) Fatalf(format string, args ...interface{}) { z.Fatalf(format, args...) } + +func (z *zapLogger) V(int) bool { return z.Enabled(zapcore.DebugLevel) } diff --git a/logger/option.go b/logger/option.go new file mode 100644 index 0000000..23d1277 --- /dev/null +++ b/logger/option.go @@ -0,0 +1,23 @@ +package logger + +import "go.uber.org/zap" + +func WithSamplingInitial(v int) Option { return func(o *options) { o.SamplingInitial = v } } + +func WithSamplingThereafter(v int) Option { return func(o *options) { o.SamplingThereafter = v } } + +func WithFormat(v string) Option { return func(o *options) { o.Format = v } } + +func WithLevel(v string) Option { return func(o *options) { o.Level = v } } + +func WithTraceLevel(v string) Option { return func(o *options) { o.TraceLevel = v } } + +func WithoutDisclaimer() Option { return func(o *options) { o.NoDisclaimer = true } } + +func WithoutCaller() Option { return func(o *options) { o.NoCaller = true } } + +func WithAppName(v string) Option { return func(o *options) { o.AppName = v } } + +func WithAppVersion(v string) Option { return func(o *options) { o.AppVersion = v } } + +func WithZapOptions(opts ...zap.Option) Option { return func(o *options) { o.Options = opts } } diff --git a/logger/zap.go b/logger/zap.go new file mode 100644 index 0000000..6228af8 --- /dev/null +++ b/logger/zap.go @@ -0,0 +1,131 @@ +package logger + +import ( + "strings" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +type ( + Option func(o *options) + + options struct { + Options []zap.Option + + SamplingInitial int + SamplingThereafter int + + Format string + Level string + TraceLevel string + + NoCaller bool + NoDisclaimer bool + + AppName string + AppVersion string + } +) + +const ( + formatJSON = "json" + formatConsole = "console" + + defaultSamplingInitial = 100 + defaultSamplingThereafter = 100 + + lvlInfo = "info" + lvlWarn = "warn" + lvlDebug = "debug" + lvlError = "error" + lvlFatal = "fatal" + lvlPanic = "panic" +) + +func safeLevel(lvl string) zap.AtomicLevel { + switch strings.ToLower(lvl) { + case lvlDebug: + return zap.NewAtomicLevelAt(zap.DebugLevel) + case lvlWarn: + return zap.NewAtomicLevelAt(zap.WarnLevel) + case lvlError: + return zap.NewAtomicLevelAt(zap.ErrorLevel) + case lvlFatal: + return zap.NewAtomicLevelAt(zap.FatalLevel) + case lvlPanic: + return zap.NewAtomicLevelAt(zap.PanicLevel) + default: + return zap.NewAtomicLevelAt(zap.InfoLevel) + } +} + +func defaults() *options { + return &options{ + SamplingInitial: defaultSamplingInitial, + SamplingThereafter: defaultSamplingThereafter, + + Format: formatConsole, + Level: lvlDebug, + TraceLevel: lvlInfo, + + NoCaller: false, + NoDisclaimer: false, + + AppName: "", + AppVersion: "", + } +} + +func New(opts ...Option) (*zap.Logger, error) { + o := defaults() + c := zap.NewProductionConfig() + + c.OutputPaths = []string{"stdout"} + c.ErrorOutputPaths = []string{"stdout"} + + for _, opt := range opts { + opt(o) + } + + // set sampling + c.Sampling = &zap.SamplingConfig{ + Initial: o.SamplingInitial, + Thereafter: o.SamplingThereafter, + } + + // logger level + c.Level = safeLevel(o.Level) + traceLvl := safeLevel(o.TraceLevel) + + // logger format + switch f := o.Format; strings.ToLower(f) { + case formatConsole: + c.Encoding = formatConsole + default: + c.Encoding = formatJSON + } + + // logger time + c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + + if o.NoCaller { + c.EncoderConfig.EncodeCaller = nil + } + + // enable trace only for current log-level + o.Options = append(o.Options, zap.AddStacktrace(traceLvl)) + + l, err := c.Build(o.Options...) + if err != nil { + return nil, err + } + + if o.NoDisclaimer { + return l, nil + } + + return l.With( + zap.String("app_name", o.AppName), + zap.String("app_version", o.AppVersion)), nil +} diff --git a/main.go b/main.go index 3fb5c6c..9e07835 100644 --- a/main.go +++ b/main.go @@ -1,50 +1,43 @@ package main import ( - "github.com/nspcc-dev/cdn-sdk/logger" "github.com/nspcc-dev/neofs-http-gate/global" + "github.com/nspcc-dev/neofs-http-gate/logger" "github.com/spf13/viper" "go.uber.org/zap" ) -func newLogger(v *viper.Viper) *zap.Logger { - options := []logger.Option{ - logger.WithLevel(v.GetString(cfgLoggerLevel)), - logger.WithTraceLevel(v.GetString(cfgLoggerTraceLevel)), - - logger.WithFormat(v.GetString(cfgLoggerFormat)), - - logger.WithSamplingInitial(v.GetInt(cfgLoggerSamplingInitial)), - logger.WithSamplingThereafter(v.GetInt(cfgLoggerSamplingThereafter)), - - logger.WithAppName(v.GetString(cfgApplicationName)), - logger.WithAppVersion(v.GetString(cfgApplicationVersion)), - } - - if v.GetBool(cfgLoggerNoCaller) { - options = append(options, logger.WithoutCaller()) - } - - if v.GetBool(cfgLoggerNoDisclaimer) { - options = append(options, logger.WithoutDisclaimer()) - } - - l, err := logger.New(options...) - if err != nil { - panic(err) - } - - return l -} - func main() { var ( v = settings() l = newLogger(v) - g = global.Context() - a = newApp(g, WithLogger(l), WithConfig(v)) ) - go a.Serve(g) - go a.Worker(g) - a.Wait() + globalContext := global.Context() + app := newApp(globalContext, WithLogger(l), WithConfig(v)) + go app.Serve(globalContext) + go app.Worker(globalContext) + app.Wait() +} + +func newLogger(v *viper.Viper) *zap.Logger { + options := []logger.Option{ + logger.WithLevel(v.GetString(cfgLoggerLevel)), + logger.WithTraceLevel(v.GetString(cfgLoggerTraceLevel)), + logger.WithFormat(v.GetString(cfgLoggerFormat)), + logger.WithSamplingInitial(v.GetInt(cfgLoggerSamplingInitial)), + logger.WithSamplingThereafter(v.GetInt(cfgLoggerSamplingThereafter)), + logger.WithAppName(v.GetString(cfgApplicationName)), + logger.WithAppVersion(v.GetString(cfgApplicationVersion)), + } + if v.GetBool(cfgLoggerNoCaller) { + options = append(options, logger.WithoutCaller()) + } + if v.GetBool(cfgLoggerNoDisclaimer) { + options = append(options, logger.WithoutDisclaimer()) + } + l, err := logger.New(options...) + if err != nil { + panic(err) + } + return l } diff --git a/neofs/client-plant.go b/neofs/client-plant.go new file mode 100644 index 0000000..5102c29 --- /dev/null +++ b/neofs/client-plant.go @@ -0,0 +1,286 @@ +package neofs + +import ( + "bytes" + "context" + "crypto/ecdsa" + "io" + "math" + "time" + + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/container" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-api-go/pkg/token" + objectCore "github.com/nspcc-dev/neofs-node/pkg/core/object" + "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/transformer" + "github.com/pkg/errors" + "google.golang.org/grpc" +) + +const ( + nodeConnectionTimeout = 10 * time.Second + maxObjectSize = uint64(1 << (20 + 6)) // 64MB +) + +type PutOptions struct { + Client client.Client + SessionToken *token.SessionToken + BearerToken *token.BearerToken + // ... + ContainerID *container.ID + OwnerID *owner.ID + PrepareObjectOnsite bool + Reader io.Reader +} + +type GetOptions struct { + Client client.Client + SessionToken *token.SessionToken + BearerToken *token.BearerToken + // ... + ObjectAddress *object.Address + Writer io.Writer +} + +type SearchOptions struct { + Client client.Client + SessionToken *token.SessionToken + BearerToken *token.BearerToken + // ... + ContainerID *container.ID + Attribute struct { + Key string + Value string + } +} + +type DeleteOptions struct { + Client client.Client + SessionToken *token.SessionToken + BearerToken *token.BearerToken + // ... + ObjectAddress *object.Address +} + +type ObjectClient interface { + Put(context.Context, *PutOptions) (*object.Address, error) + Get(context.Context, *GetOptions) (*object.Object, error) + Search(context.Context, *SearchOptions) ([]*object.ID, error) + Delete(context.Context, *DeleteOptions) error +} + +type ClientPlant interface { + GetReusableArtifacts(ctx context.Context) (client.Client, *token.SessionToken, error) + Object() ObjectClient + OwnerID() *owner.ID +} + +type objectClient struct { + key *ecdsa.PrivateKey + conn *grpc.ClientConn +} + +type neofsClient struct { + key *ecdsa.PrivateKey + ownerID *owner.ID + conn *grpc.ClientConn +} + +func (cc *neofsClient) GetReusableArtifacts(ctx context.Context) (client.Client, *token.SessionToken, error) { + c, err := client.New(client.WithDefaultPrivateKey(cc.key), client.WithGRPCConnection(cc.conn)) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create reusable neofs client") + } + st, err := c.CreateSession(ctx, math.MaxUint64) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create reusable neofs session token") + } + return c, st, nil +} + +func (cc *neofsClient) Object() ObjectClient { + return &objectClient{key: cc.key, conn: cc.conn} +} + +func (cc *neofsClient) OwnerID() *owner.ID { + return cc.ownerID +} + +func NewClientPlant(ctx context.Context, address string, creds Credentials) (ClientPlant, error) { + toctx, c := context.WithTimeout(ctx, nodeConnectionTimeout) + defer c() + conn, err := grpc.DialContext(toctx, address, grpc.WithInsecure(), grpc.WithBlock()) + if err != nil { + if err == context.DeadlineExceeded { + err = errors.New("failed to connect to neofs node") + } + return nil, err + } + return &neofsClient{ + key: creds.PrivateKey(), + ownerID: creds.Owner(), + conn: conn, + }, nil +} + +func (oc *objectClient) Put(ctx context.Context, options *PutOptions) (*object.Address, error) { + var ( + err error + objectID *object.ID + ) + address := object.NewAddress() + if options.PrepareObjectOnsite { + rawObject := objectCore.NewRaw() + rawObject.SetContainerID(options.ContainerID) + rawObject.SetOwnerID(options.OwnerID) + ns := newNetworkState(ctx, options.Client) + objectTarget := transformer.NewPayloadSizeLimiter(maxObjectSize, func() transformer.ObjectTarget { + return transformer.NewFormatTarget(&transformer.FormatterParams{ + Key: oc.key, + NextTarget: &remoteClientTarget{ + ctx: ctx, + client: options.Client, + }, + NetworkState: ns, + }) + }) + if err = ns.LastError(); err != nil { + return nil, err + } + err = objectTarget.WriteHeader(rawObject) + if err != nil { + return nil, err + } + _, err = io.Copy(objectTarget, options.Reader) + if err != nil { + return nil, err + } + var ids *transformer.AccessIdentifiers + ids, err = objectTarget.Close() + if err != nil { + return nil, err + } + address.SetObjectID(ids.SelfID()) + } else { + rawObject := object.NewRaw() + rawObject.SetContainerID(options.ContainerID) + rawObject.SetOwnerID(options.OwnerID) + ops := new(client.PutObjectParams). + WithObject(rawObject.Object()). + WithPayloadReader(options.Reader) + objectID, err = options.Client.PutObject( + ctx, + ops, + client.WithSession(options.SessionToken), + client.WithBearer(options.BearerToken), + ) + if err != nil { + return nil, err + } + address.SetObjectID(objectID) + } + address.SetContainerID(options.ContainerID) + return address, nil +} + +func (oc *objectClient) Get(ctx context.Context, options *GetOptions) (*object.Object, error) { + var ( + err error + obj *object.Object + ) + ops := new(client.GetObjectParams). + WithAddress(options.ObjectAddress). + WithPayloadWriter(options.Writer) + obj, err = options.Client.GetObject( + ctx, + ops, + client.WithSession(options.SessionToken), + client.WithBearer(options.BearerToken), + ) + return obj, err +} + +func (oc *objectClient) Search(ctx context.Context, options *SearchOptions) ([]*object.ID, error) { + sfs := object.NewSearchFilters() + sfs.AddRootFilter() + sfs.AddFilter(options.Attribute.Key, options.Attribute.Value, object.MatchStringEqual) + sops := new(client.SearchObjectParams) + sops.WithContainerID(options.ContainerID) + sops.WithSearchFilters(sfs) + return options.Client.SearchObject( + ctx, + sops, + client.WithSession(options.SessionToken), + client.WithBearer(options.BearerToken), + ) +} + +func (oc *objectClient) Delete(ctx context.Context, options *DeleteOptions) error { + ops := new(client.DeleteObjectParams).WithAddress(options.ObjectAddress) + err := options.Client.DeleteObject( + ctx, + ops, + client.WithSession(options.SessionToken), + client.WithBearer(options.BearerToken), + ) + return err +} + +type remoteClientTarget struct { + ctx context.Context + client client.Client + object *object.Object + payload []byte +} + +func (rct *remoteClientTarget) WriteHeader(raw *objectCore.RawObject) error { + rct.object = raw.Object().SDK() + return nil +} + +func (rct *remoteClientTarget) Write(p []byte) (n int, err error) { + rct.payload = append(rct.payload, p...) + return len(p), nil +} + +func (rct *remoteClientTarget) Close() (*transformer.AccessIdentifiers, error) { + id, err := rct.client.PutObject( + rct.ctx, new(client.PutObjectParams). + WithObject(rct.object). + WithPayloadReader(bytes.NewReader(rct.payload)), + ) + if err != nil { + return nil, err + } + return new(transformer.AccessIdentifiers).WithSelfID(id), nil +} + +type networkState struct { + ctx context.Context + client client.Client + lastError error + onError func(error) +} + +func newNetworkState(ctx context.Context, client client.Client) *networkState { + ns := &networkState{ + ctx: ctx, + client: client, + } + ns.onError = func(err error) { ns.lastError = err } + return ns +} + +func (ns *networkState) LastError() error { + return ns.lastError +} + +func (ns *networkState) CurrentEpoch() uint64 { + ce, err := ns.client.NetworkInfo(ns.ctx) + if err != nil { + ns.onError(err) + } + return ce.CurrentEpoch() +} diff --git a/neofs/credentials.go b/neofs/credentials.go new file mode 100644 index 0000000..48faf64 --- /dev/null +++ b/neofs/credentials.go @@ -0,0 +1,67 @@ +package neofs + +import ( + "crypto/ecdsa" + + "github.com/nspcc-dev/neofs-api-go/pkg/owner" + crypto "github.com/nspcc-dev/neofs-crypto" +) + +type ( + // Credentials contains methods that needed to work with NeoFS. + Credentials interface { + WIF() string + Owner() *owner.ID + PublicKey() *ecdsa.PublicKey + PrivateKey() *ecdsa.PrivateKey + } + + cred struct { + key *ecdsa.PrivateKey + owner *owner.ID + wif string + } +) + +// New creates an instance of Credentials through string representation of secret. +// It allows passing WIF, path, hex-encoded and others. +func NewCredentials(secret string) (Credentials, error) { + key, err := crypto.LoadPrivateKey(secret) + if err != nil { + return nil, err + } + return setFromPrivateKey(key) +} + +// PrivateKey returns ecdsa.PrivateKey. +func (c *cred) PrivateKey() *ecdsa.PrivateKey { + return c.key +} + +// PublicKey returns ecdsa.PublicKey. +func (c *cred) PublicKey() *ecdsa.PublicKey { + return &c.key.PublicKey +} + +// Owner returns owner.ID. +func (c *cred) Owner() *owner.ID { + return c.owner +} + +// WIF returns string representation of WIF. +func (c *cred) WIF() string { + return c.wif +} + +func setFromPrivateKey(key *ecdsa.PrivateKey) (*cred, error) { + wallet, err := owner.NEO3WalletFromPublicKey(&key.PublicKey) + if err != nil { + return nil, err + } + ownerID := owner.NewIDFromNeo3Wallet(wallet) + wif, err := crypto.WIFEncode(key) + if err != nil { + return nil, err + } + return &cred{key: key, owner: ownerID, wif: wif}, nil +} diff --git a/pool/pool.go b/pool/pool.go new file mode 100644 index 0000000..94874ec --- /dev/null +++ b/pool/pool.go @@ -0,0 +1,39 @@ +package pool + +import ( + "context" + + "github.com/nspcc-dev/neofs-api-go/pkg/token" + "google.golang.org/grpc" +) + +type Client interface { + // receive status of connection pool + Status() error + // worker should be run in goroutine to re-balancing + Worker(context.Context) + Connection(context.Context) (*grpc.ClientConn, error) + Session(context.Context, *grpc.ClientConn) (*token.SessionToken, error) +} + +type pool struct{} + +func (p *pool) Status() error { + return nil +} + +func (p *pool) Worker(ctx context.Context) { + panic("not implemented") +} + +func (p *pool) Connection(ctx context.Context) (*grpc.ClientConn, error) { + panic("not implemented") +} + +func (p *pool) Session(ctx context.Context, conn *grpc.ClientConn) (*token.SessionToken, error) { + panic("not implemented") +} + +func New() Client { + return &pool{} +} diff --git a/receive.go b/receive.go index ae1b555..9728e7b 100644 --- a/receive.go +++ b/receive.go @@ -9,9 +9,9 @@ import ( "sync" "time" - sdk "github.com/nspcc-dev/cdn-sdk" "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-http-gate/neofs" "github.com/pkg/errors" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -31,7 +31,7 @@ type ( *fasthttp.RequestCtx log *zap.Logger - obj sdk.ObjectClient + obj neofs.ObjectClient } objectIDs []*object.ID @@ -49,40 +49,38 @@ func (d *detector) Write(data []byte) (int, error) { return d.Writer.Write(data) } -func (r *request) receiveFile(address *object.Address) { +func (r *request) receiveFile(options *neofs.GetOptions) { var ( err error dis = "inline" start = time.Now() filename string ) - - if err = checkAndPropagateBearerToken(r.RequestCtx); err != nil { - r.log.Error("could not fetch bearer token", zap.Error(err)) - r.Error("could not fetch bearer token", fasthttp.StatusBadRequest) + if err = storeBearerToken(r.RequestCtx); err != nil { + r.log.Error("could not fetch and store bearer token", zap.Error(err)) + r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) return } - writer := newDetector(r.Response.BodyWriter()) - obj, err := r.obj.Get(r, address, sdk.WithGetWriter(writer)) + // obj, err := r.obj.Get(r, address, sdk.WithGetWriter(writer)) + options.Writer = writer + obj, err := r.obj.Get(r.RequestCtx, options) if err != nil { - r.log.Error("could not receive object", + r.log.Error( + "could not receive object", zap.Stringer("elapsed", time.Since(start)), - zap.Error(err)) - + zap.Error(err), + ) var ( msg = errors.Wrap(err, "could not receive object").Error() code = fasthttp.StatusBadRequest ) - if st, ok := status.FromError(errors.Cause(err)); ok && st != nil { if st.Code() == codes.NotFound { code = fasthttp.StatusNotFound } - msg = st.Message() } - r.Error(msg, code) return } @@ -139,68 +137,79 @@ func (a *app) request(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { RequestCtx: ctx, log: log, - obj: a.cli.Object(), + obj: a.plant.Object(), } } func (a *app) byAddress(c *fasthttp.RequestCtx) { var ( - err error - adr = object.NewAddress() - cid, _ = c.UserValue("cid").(string) - oid, _ = c.UserValue("oid").(string) - val = strings.Join([]string{cid, oid}, "/") - log = a.log.With( - zap.String("cid", cid), - zap.String("oid", oid)) + err error + address = object.NewAddress() + cid, _ = c.UserValue("cid").(string) + oid, _ = c.UserValue("oid").(string) + val = strings.Join([]string{cid, oid}, "/") + log = a.log.With(zap.String("cid", cid), zap.String("oid", oid)) ) - - if err = adr.Parse(val); err != nil { + if err = address.Parse(val); err != nil { log.Error("wrong object address", zap.Error(err)) c.Error("wrong object address", fasthttp.StatusBadRequest) return } - - a.request(c, log).receiveFile(adr) + // TODO: Take this from a sync-pool. + getOpts := new(neofs.GetOptions) + getOpts.Client = a.getOperations.client + getOpts.SessionToken = a.getOperations.sessionToken + getOpts.ObjectAddress = address + getOpts.Writer = nil + a.request(c, log).receiveFile(getOpts) } func (a *app) byAttribute(c *fasthttp.RequestCtx) { var ( err error - ids []*object.ID - cid = container.NewID() - adr = object.NewAddress() - sCID, _ = c.UserValue("cid").(string) + scid, _ = c.UserValue("cid").(string) key, _ = c.UserValue("attr_key").(string) val, _ = c.UserValue("attr_val").(string) - - log = a.log.With( - zap.String("cid", sCID), - zap.String("attr_key", key), - zap.String("attr_val", val)) + log = a.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ) - - if err = cid.Parse(sCID); err != nil { + cid := container.NewID() + if err = cid.Parse(scid); err != nil { log.Error("wrong container id", zap.Error(err)) c.Error("wrong container id", fasthttp.StatusBadRequest) return - } else if ids, err = a.cli.Object().Search(c, cid, sdk.SearchRootObjects(), sdk.SearchByAttribute(key, val)); err != nil { + // } else if ids, err = a.cli.Object().Search(c, cid, sdk.SearchRootObjects(), sdk.SearchByAttribute(key, val)); err != nil { + } + // TODO: Take this from a sync-pool. + searchOpts := new(neofs.SearchOptions) + searchOpts.Client = a.getOperations.client + searchOpts.SessionToken = a.getOperations.sessionToken + searchOpts.BearerToken = nil + searchOpts.ContainerID = cid + searchOpts.Attribute.Key = key + searchOpts.Attribute.Value = val + var ids []*object.ID + if ids, err = a.plant.Object().Search(c, searchOpts); err != nil { log.Error("something went wrong", zap.Error(err)) c.Error("something went wrong", fasthttp.StatusBadRequest) return } else if len(ids) == 0 { log.Debug("object not found") - c.Error("not found", fasthttp.StatusNotFound) + c.Error("object not found", fasthttp.StatusNotFound) return } - if len(ids) > 1 { log.Debug("found multiple objects", zap.Strings("object_ids", objectIDs(ids).Slice()), zap.Stringer("show_object_id", ids[0])) } - - adr.SetContainerID(cid) - adr.SetObjectID(ids[0]) - a.request(c, log).receiveFile(adr) + address := object.NewAddress() + address.SetContainerID(cid) + address.SetObjectID(ids[0]) + // TODO: Take this from a sync-pool. + getOpts := new(neofs.GetOptions) + getOpts.Client = a.getOperations.client + getOpts.SessionToken = a.getOperations.sessionToken + getOpts.ObjectAddress = address + getOpts.Writer = nil + a.request(c, log).receiveFile(getOpts) } diff --git a/settings.go b/settings.go index 206fd0a..0e5f79f 100644 --- a/settings.go +++ b/settings.go @@ -79,11 +79,9 @@ var ignore = map[string]struct{}{ cfgApplicationName: {}, cfgApplicationVersion: {}, cfgApplicationBuildTime: {}, - - cfgPeers: {}, - - cmdHelp: {}, - cmdVersion: {}, + cfgPeers: {}, + cmdHelp: {}, + cmdVersion: {}, } func (empty) Read([]byte) (int, error) { return 0, io.EOF } @@ -106,14 +104,14 @@ func settings() *viper.Viper { help := flags.BoolP(cmdHelp, "h", false, "show help") version := flags.BoolP(cmdVersion, "v", false, "show version") - flags.String(cmdNeoFSKey, "", `"Path to private key file, hex string or wif`) + flags.String(cmdNeoFSKey, "", `path to private key file, hex string or wif`) flags.Bool(cmdVerbose, false, "debug gRPC connections") flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout") flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout") flags.Duration(cfgRebalance, defaultRebalanceTimer, "gRPC connection rebalance timer") - flags.String(cfgListenAddress, "0.0.0.0:8082", "HTTP Gateway listen address") + flags.String(cfgListenAddress, "0.0.0.0:8082", "address to listen") peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") // set prefers: diff --git a/upload.go b/upload.go index 1360ec5..20a65f5 100644 --- a/upload.go +++ b/upload.go @@ -7,10 +7,11 @@ import ( "strconv" "time" - sdk "github.com/nspcc-dev/cdn-sdk" "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-http-gate/neofs" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -29,18 +30,17 @@ func newPutResponse(addr *object.Address) *putResponse { } } -func (pr *putResponse) Encode(w io.Writer) error { +func (pr *putResponse) encode(w io.Writer) error { enc := json.NewEncoder(w) enc.SetIndent("", "\t") return enc.Encode(pr) } -func (a *app) fetchOwner(ctx context.Context) *owner.ID { - if tkn, err := sdk.BearerToken(ctx); err == nil && tkn != nil { - return tkn.Issuer() +func (a *app) fetchOwnerAndBearerToken(ctx context.Context) (*owner.ID, *token.BearerToken) { + if token, err := loadBearerToken(ctx); err == nil && token != nil { + return token.Issuer(), token } - - return a.cli.Owner() + return a.plant.OwnerID(), nil } func (a *app) upload(c *fasthttp.RequestCtx) { @@ -50,94 +50,93 @@ func (a *app) upload(c *fasthttp.RequestCtx) { addr *object.Address cid = container.NewID() sCID, _ = c.UserValue("cid").(string) - - log = a.log.With(zap.String("cid", sCID)) + log = a.log.With(zap.String("cid", sCID)) ) - - if err = checkAndPropagateBearerToken(c); err != nil { + if err = storeBearerToken(c); err != nil { log.Error("could not fetch bearer token", zap.Error(err)) c.Error("could not fetch bearer token", fasthttp.StatusBadRequest) return } - if err = cid.Parse(sCID); err != nil { log.Error("wrong container id", zap.Error(err)) c.Error("wrong container id", fasthttp.StatusBadRequest) return } - defer func() { // if temporary reader can be closed - close it if file == nil { return } - - log.Debug("close temporary multipart/form file", + err := file.Close() + log.Debug( + "close temporary multipart/form file", zap.Stringer("address", addr), zap.String("filename", file.FileName()), - zap.Error(file.Close())) + zap.Error(err), + ) }() - boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(a.log, c.RequestBodyStream(), boundary); err != nil { log.Error("could not receive multipart/form", zap.Error(err)) c.Error("could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) - return } - filtered := filterHeaders(a.log, &c.Request.Header) attributes := make([]*object.Attribute, 0, len(filtered)) - // prepares attributes from filtered headers for key, val := range filtered { attribute := object.NewAttribute() attribute.SetKey(key) attribute.SetValue(val) - attributes = append(attributes, attribute) } - // sets FileName attribute if it wasn't set from header if _, ok := filtered[object.AttributeFileName]; !ok { filename := object.NewAttribute() filename.SetKey(object.AttributeFileName) filename.SetValue(file.FileName()) - attributes = append(attributes, filename) } - // sets Timestamp attribute if it wasn't set from header and enabled by settings if _, ok := filtered[object.AttributeTimestamp]; !ok && a.enableDefaultTimestamp { timestamp := object.NewAttribute() timestamp.SetKey(object.AttributeTimestamp) timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) - attributes = append(attributes, timestamp) } - + oid, bt := a.fetchOwnerAndBearerToken(c) // prepares new object and fill it raw := object.NewRaw() raw.SetContainerID(cid) - raw.SetOwnerID(a.fetchOwner(c)) + raw.SetOwnerID(oid) raw.SetAttributes(attributes...) - // tries to put file into NeoFS or throw error - if addr, err = a.cli.Object().Put(c, raw.Object(), sdk.WithPutReader(file)); err != nil { - log.Error("could not store file in NeoFS", zap.Error(err)) - c.Error("could not store file in NeoFS", fasthttp.StatusBadRequest) - + // if addr, err = a.plant.Object().Put(c, raw.Object(), sdk.WithPutReader(file)); err != nil { + // TODO: Take this from a sync pool. + putOpts := new(neofs.PutOptions) + putOpts.Client, putOpts.SessionToken, err = a.plant.GetReusableArtifacts(c) + if err != nil { + log.Error("failed to get neofs client's reusable artifacts", zap.Error(err)) + c.Error("failed to get neofs client's reusable artifacts", fasthttp.StatusInternalServerError) + return + } + putOpts.BearerToken = bt + putOpts.ContainerID = cid + putOpts.OwnerID = oid + putOpts.PrepareObjectOnsite = false + putOpts.Reader = file + if addr, err = a.plant.Object().Put(c, putOpts); err != nil { + log.Error("could not store file in NeoFS", zap.Error(err)) + c.Error("could not store file in NeoFS", fasthttp.StatusBadRequest) return } - // tries to return response, otherwise, if something went wrong throw error - if err = newPutResponse(addr).Encode(c); err != nil { + if err = newPutResponse(addr).encode(c); err != nil { log.Error("could not prepare response", zap.Error(err)) c.Error("could not prepare response", fasthttp.StatusBadRequest) return } - // reports status code and content type c.Response.SetStatusCode(fasthttp.StatusOK) c.Response.Header.SetContentType(jsonHeader) From eb92219e1440cc953d9021d0b9750d921e237fcd Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Wed, 31 Mar 2021 19:58:42 +0300 Subject: [PATCH 114/548] [#19] Extract uploading logic into a separate package Signed-off-by: Pavel Korotkov --- app.go | 4 +- go_dev.mod | 26 -- neofs/client-plant.go | 24 +- receive.go | 22 +- bearer.go => tokens/bearer-token.go | 26 +- bearer_test.go => tokens/bearer-token_test.go | 10 +- filter.go => uploader/filter.go | 2 +- multipart.go => uploader/multipart.go | 2 +- upload.go => uploader/upload.go | 225 +++++++++--------- 9 files changed, 153 insertions(+), 188 deletions(-) delete mode 100644 go_dev.mod rename bearer.go => tokens/bearer-token.go (84%) rename bearer_test.go => tokens/bearer-token_test.go (92%) rename filter.go => uploader/filter.go (98%) rename multipart.go => uploader/multipart.go (97%) rename upload.go => uploader/upload.go (77%) diff --git a/app.go b/app.go index 8fe8960..f1f1336 100644 --- a/app.go +++ b/app.go @@ -10,6 +10,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-http-gate/logger" "github.com/nspcc-dev/neofs-http-gate/neofs" + "github.com/nspcc-dev/neofs-http-gate/uploader" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -151,10 +152,11 @@ func (a *app) Serve(ctx context.Context) { a.log.Info("shutting down web server", zap.Error(a.web.Shutdown())) close(a.webDone) }() + uploader := uploader.New(a.log, a.plant, a.enableDefaultTimestamp) // Configure router. r := router.New() r.RedirectTrailingSlash = true - r.POST("/upload/{cid}", a.upload) + r.POST("/upload/{cid}", uploader.Upload) a.log.Info("added path /upload/{cid}") r.GET("/get/{cid}/{oid}", a.byAddress) a.log.Info("added path /get/{cid}/{oid}") diff --git a/go_dev.mod b/go_dev.mod deleted file mode 100644 index f45f6ae..0000000 --- a/go_dev.mod +++ /dev/null @@ -1,26 +0,0 @@ -module github.com/nspcc-dev/neofs-http-gate - -go 1.16 - -require ( - github.com/fasthttp/router v0.6.1 - github.com/nspcc-dev/neofs-api v0.0.0-00000000000000-000000000000 - github.com/nspcc-dev/neofs-crypto v0.2.3 - github.com/prometheus/client_golang v1.4.1 // v1.2.1 => v1.4.1 - github.com/prometheus/common v0.9.1 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.6.2 // v1.6.1 => v1.6.2 - github.com/valyala/fasthttp v1.9.0 - go.uber.org/atomic v1.6.0 - go.uber.org/zap v1.14.0 - golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 // indirect - golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect - golang.org/x/text v0.3.2 // indirect - google.golang.org/grpc v1.27.1 -) - -// For debug reasons -replace ( - github.com/nspcc-dev/neofs-api => ../neofs-api - google.golang.org/grpc => ../grpc-go -) diff --git a/neofs/client-plant.go b/neofs/client-plant.go index 5102c29..cf3aaa9 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -21,14 +21,17 @@ import ( const ( nodeConnectionTimeout = 10 * time.Second - maxObjectSize = uint64(1 << (20 + 6)) // 64MB + maxObjectSize = uint64(1 << 26) // 64MiB ) -type PutOptions struct { +type BaseOptions struct { Client client.Client SessionToken *token.SessionToken BearerToken *token.BearerToken - // ... +} + +type PutOptions struct { + BaseOptions ContainerID *container.ID OwnerID *owner.ID PrepareObjectOnsite bool @@ -36,19 +39,13 @@ type PutOptions struct { } type GetOptions struct { - Client client.Client - SessionToken *token.SessionToken - BearerToken *token.BearerToken - // ... + BaseOptions ObjectAddress *object.Address Writer io.Writer } type SearchOptions struct { - Client client.Client - SessionToken *token.SessionToken - BearerToken *token.BearerToken - // ... + BaseOptions ContainerID *container.ID Attribute struct { Key string @@ -57,10 +54,7 @@ type SearchOptions struct { } type DeleteOptions struct { - Client client.Client - SessionToken *token.SessionToken - BearerToken *token.BearerToken - // ... + BaseOptions ObjectAddress *object.Address } diff --git a/receive.go b/receive.go index 9728e7b..25199f8 100644 --- a/receive.go +++ b/receive.go @@ -12,6 +12,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-http-gate/neofs" + "github.com/nspcc-dev/neofs-http-gate/tokens" "github.com/pkg/errors" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -23,13 +24,11 @@ type ( detector struct { io.Writer sync.Once - contentType string } request struct { *fasthttp.RequestCtx - log *zap.Logger obj neofs.ObjectClient } @@ -45,7 +44,6 @@ func (d *detector) Write(data []byte) (int, error) { d.Once.Do(func() { d.contentType = http.DetectContentType(data) }) - return d.Writer.Write(data) } @@ -56,13 +54,12 @@ func (r *request) receiveFile(options *neofs.GetOptions) { start = time.Now() filename string ) - if err = storeBearerToken(r.RequestCtx); err != nil { + if err = tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) return } writer := newDetector(r.Response.BodyWriter()) - // obj, err := r.obj.Get(r, address, sdk.WithGetWriter(writer)) options.Writer = writer obj, err := r.obj.Get(r.RequestCtx, options) if err != nil { @@ -84,22 +81,17 @@ func (r *request) receiveFile(options *neofs.GetOptions) { r.Error(msg, code) return } - if r.Request.URI().QueryArgs().GetBool("download") { dis = "attachment" } - r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) r.Response.Header.Set("x-object-id", obj.ID().String()) r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) r.Response.Header.Set("x-container-id", obj.ContainerID().String()) - for _, attr := range obj.Attributes() { key := attr.Key() val := attr.Value() - r.Response.Header.Set("x-"+key, val) - switch key { case object.AttributeFileName: filename = val @@ -112,13 +104,10 @@ func (r *request) receiveFile(options *neofs.GetOptions) { zap.Error(err)) continue } - r.Response.Header.Set("Last-Modified", time.Unix(value, 0).Format(time.RFC1123)) } - } - r.SetContentType(writer.contentType) r.Response.Header.Set("Content-Disposition", dis+"; filename="+path.Base(filename)) } @@ -128,16 +117,14 @@ func (o objectIDs) Slice() []string { for _, oid := range o { res = append(res, oid.String()) } - return res } func (a *app) request(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { return &request{ RequestCtx: ctx, - - log: log, - obj: a.plant.Object(), + log: log, + obj: a.plant.Object(), } } @@ -177,7 +164,6 @@ func (a *app) byAttribute(c *fasthttp.RequestCtx) { log.Error("wrong container id", zap.Error(err)) c.Error("wrong container id", fasthttp.StatusBadRequest) return - // } else if ids, err = a.cli.Object().Search(c, cid, sdk.SearchRootObjects(), sdk.SearchByAttribute(key, val)); err != nil { } // TODO: Take this from a sync-pool. searchOpts := new(neofs.SearchOptions) diff --git a/bearer.go b/tokens/bearer-token.go similarity index 84% rename from bearer.go rename to tokens/bearer-token.go index e14c35c..337c604 100644 --- a/bearer.go +++ b/tokens/bearer-token.go @@ -1,4 +1,4 @@ -package main +package tokens import ( "bytes" @@ -25,20 +25,18 @@ const ( // return // } -func fromHeader(h *fasthttp.RequestHeader) []byte { +func BearerTokenFromHeader(h *fasthttp.RequestHeader) []byte { auth := h.Peek(fasthttp.HeaderAuthorization) if auth == nil || !bytes.HasPrefix(auth, []byte(bearerTokenHdr)) { return nil } - if auth = bytes.TrimPrefix(auth, []byte(bearerTokenHdr+" ")); len(auth) == 0 { return nil } - return auth } -func fromCookie(h *fasthttp.RequestHeader) []byte { +func BearerTokenFromCookie(h *fasthttp.RequestHeader) []byte { auth := h.Cookie(bearerTokenHdr) if len(auth) == 0 { return nil @@ -47,7 +45,7 @@ func fromCookie(h *fasthttp.RequestHeader) []byte { return auth } -func storeBearerToken(ctx *fasthttp.RequestCtx) error { +func StoreBearerToken(ctx *fasthttp.RequestCtx) error { tkn, err := fetchBearerToken(ctx) if err != nil { return err @@ -57,6 +55,13 @@ func storeBearerToken(ctx *fasthttp.RequestCtx) error { return nil } +func LoadBearerToken(ctx context.Context) (*token.BearerToken, error) { + if tkn, ok := ctx.Value(bearerTokenKey).(*token.BearerToken); ok && tkn != nil { + return tkn, nil + } + return nil, errors.New("found empty bearer token") +} + func fetchBearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { // ignore empty value if ctx == nil { @@ -68,7 +73,7 @@ func fetchBearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { buf []byte tkn = new(token.BearerToken) ) - for _, parse := range []fromHandler{fromHeader, fromCookie} { + for _, parse := range []fromHandler{BearerTokenFromHeader, BearerTokenFromCookie} { if buf = parse(&ctx.Request.Header); buf == nil { continue } else if data, err := base64.StdEncoding.DecodeString(string(buf)); err != nil { @@ -86,10 +91,3 @@ func fetchBearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { return nil, lastErr } - -func loadBearerToken(ctx context.Context) (*token.BearerToken, error) { - if tkn, ok := ctx.Value(bearerTokenKey).(*token.BearerToken); ok && tkn != nil { - return tkn, nil - } - return nil, errors.New("found empty bearer token") -} diff --git a/bearer_test.go b/tokens/bearer-token_test.go similarity index 92% rename from bearer_test.go rename to tokens/bearer-token_test.go index 4cfa31f..c6ee703 100644 --- a/bearer_test.go +++ b/tokens/bearer-token_test.go @@ -1,4 +1,4 @@ -package main +package tokens import ( "encoding/base64" @@ -36,7 +36,7 @@ func Test_fromCookie(t *testing.T) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - require.Equal(t, tt.expect, fromCookie(makeTestCookie(tt.actual))) + require.Equal(t, tt.expect, BearerTokenFromCookie(makeTestCookie(tt.actual))) }) } } @@ -53,7 +53,7 @@ func Test_fromHeader(t *testing.T) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - require.Equal(t, tt.expect, fromHeader(makeTestHeader(tt.actual))) + require.Equal(t, tt.expect, BearerTokenFromHeader(makeTestHeader(tt.actual))) }) } } @@ -151,10 +151,10 @@ func Test_checkAndPropagateBearerToken(t *testing.T) { ctx := makeTestRequest(t64, "") // Expect to see the token within the context. - require.NoError(t, storeBearerToken(ctx)) + require.NoError(t, StoreBearerToken(ctx)) // Expect to see the same token without errors. - actual, err := loadBearerToken(ctx) + actual, err := LoadBearerToken(ctx) require.NoError(t, err) require.Equal(t, tkn, actual) } diff --git a/filter.go b/uploader/filter.go similarity index 98% rename from filter.go rename to uploader/filter.go index 7d8b24d..47c789a 100644 --- a/filter.go +++ b/uploader/filter.go @@ -1,4 +1,4 @@ -package main +package uploader import ( "bytes" diff --git a/multipart.go b/uploader/multipart.go similarity index 97% rename from multipart.go rename to uploader/multipart.go index f12442f..8abc6d0 100644 --- a/multipart.go +++ b/uploader/multipart.go @@ -1,4 +1,4 @@ -package main +package uploader import ( "io" diff --git a/upload.go b/uploader/upload.go similarity index 77% rename from upload.go rename to uploader/upload.go index 20a65f5..ad03b0d 100644 --- a/upload.go +++ b/uploader/upload.go @@ -1,4 +1,4 @@ -package main +package uploader import ( "context" @@ -12,10 +12,127 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-http-gate/neofs" + "github.com/nspcc-dev/neofs-http-gate/tokens" "github.com/valyala/fasthttp" "go.uber.org/zap" ) +type Uploader struct { + log *zap.Logger + plant neofs.ClientPlant + enableDefaultTimestamp bool +} + +func New(log *zap.Logger, plant neofs.ClientPlant, enableDefaultTimestamp bool) *Uploader { + return &Uploader{log, plant, enableDefaultTimestamp} +} + +func (u *Uploader) Upload(c *fasthttp.RequestCtx) { + var ( + err error + file MultipartFile + addr *object.Address + cid = container.NewID() + scid, _ = c.UserValue("cid").(string) + log = u.log.With(zap.String("cid", scid)) + ) + if err = tokens.StoreBearerToken(c); err != nil { + log.Error("could not fetch bearer token", zap.Error(err)) + c.Error("could not fetch bearer token", fasthttp.StatusBadRequest) + return + } + if err = cid.Parse(scid); err != nil { + log.Error("wrong container id", zap.Error(err)) + c.Error("wrong container id", fasthttp.StatusBadRequest) + return + } + defer func() { + // if temporary reader can be closed - close it + if file == nil { + return + } + err := file.Close() + log.Debug( + "close temporary multipart/form file", + zap.Stringer("address", addr), + zap.String("filename", file.FileName()), + zap.Error(err), + ) + }() + boundary := string(c.Request.Header.MultipartFormBoundary()) + if file, err = fetchMultipartFile(u.log, c.RequestBodyStream(), boundary); err != nil { + log.Error("could not receive multipart/form", zap.Error(err)) + c.Error("could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) + return + } + filtered := filterHeaders(u.log, &c.Request.Header) + attributes := make([]*object.Attribute, 0, len(filtered)) + // prepares attributes from filtered headers + for key, val := range filtered { + attribute := object.NewAttribute() + attribute.SetKey(key) + attribute.SetValue(val) + attributes = append(attributes, attribute) + } + // sets FileName attribute if it wasn't set from header + if _, ok := filtered[object.AttributeFileName]; !ok { + filename := object.NewAttribute() + filename.SetKey(object.AttributeFileName) + filename.SetValue(file.FileName()) + attributes = append(attributes, filename) + } + // sets Timestamp attribute if it wasn't set from header and enabled by settings + if _, ok := filtered[object.AttributeTimestamp]; !ok && u.enableDefaultTimestamp { + timestamp := object.NewAttribute() + timestamp.SetKey(object.AttributeTimestamp) + timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) + attributes = append(attributes, timestamp) + } + oid, bt := u.fetchOwnerAndBearerToken(c) + // prepares new object and fill it + raw := object.NewRaw() + raw.SetContainerID(cid) + raw.SetOwnerID(oid) + raw.SetAttributes(attributes...) + // tries to put file into NeoFS or throw error + // if addr, err = a.plant.Object().Put(c, raw.Object(), sdk.WithPutReader(file)); err != nil { + // TODO: Take this from a sync pool. + putOpts := new(neofs.PutOptions) + putOpts.Client, putOpts.SessionToken, err = u.plant.GetReusableArtifacts(c) + if err != nil { + log.Error("failed to get neofs client's reusable artifacts", zap.Error(err)) + c.Error("failed to get neofs client's reusable artifacts", fasthttp.StatusInternalServerError) + return + } + putOpts.BearerToken = bt + putOpts.ContainerID = cid + putOpts.OwnerID = oid + putOpts.PrepareObjectOnsite = false + putOpts.Reader = file + if addr, err = u.plant.Object().Put(c, putOpts); err != nil { + log.Error("could not store file in NeoFS", zap.Error(err)) + c.Error("could not store file in NeoFS", fasthttp.StatusBadRequest) + return + } + // tries to return response, otherwise, if something went wrong throw error + if err = newPutResponse(addr).encode(c); err != nil { + log.Error("could not prepare response", zap.Error(err)) + c.Error("could not prepare response", fasthttp.StatusBadRequest) + + return + } + // reports status code and content type + c.Response.SetStatusCode(fasthttp.StatusOK) + c.Response.Header.SetContentType(jsonHeader) +} + +func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*owner.ID, *token.BearerToken) { + if token, err := tokens.LoadBearerToken(ctx); err == nil && token != nil { + return token.Issuer(), token + } + return u.plant.OwnerID(), nil +} + type putResponse struct { OID string `json:"object_id"` CID string `json:"container_id"` @@ -35,109 +152,3 @@ func (pr *putResponse) encode(w io.Writer) error { enc.SetIndent("", "\t") return enc.Encode(pr) } - -func (a *app) fetchOwnerAndBearerToken(ctx context.Context) (*owner.ID, *token.BearerToken) { - if token, err := loadBearerToken(ctx); err == nil && token != nil { - return token.Issuer(), token - } - return a.plant.OwnerID(), nil -} - -func (a *app) upload(c *fasthttp.RequestCtx) { - var ( - err error - file MultipartFile - addr *object.Address - cid = container.NewID() - sCID, _ = c.UserValue("cid").(string) - log = a.log.With(zap.String("cid", sCID)) - ) - if err = storeBearerToken(c); err != nil { - log.Error("could not fetch bearer token", zap.Error(err)) - c.Error("could not fetch bearer token", fasthttp.StatusBadRequest) - return - } - if err = cid.Parse(sCID); err != nil { - log.Error("wrong container id", zap.Error(err)) - c.Error("wrong container id", fasthttp.StatusBadRequest) - return - } - defer func() { - // if temporary reader can be closed - close it - if file == nil { - return - } - err := file.Close() - log.Debug( - "close temporary multipart/form file", - zap.Stringer("address", addr), - zap.String("filename", file.FileName()), - zap.Error(err), - ) - }() - boundary := string(c.Request.Header.MultipartFormBoundary()) - if file, err = fetchMultipartFile(a.log, c.RequestBodyStream(), boundary); err != nil { - log.Error("could not receive multipart/form", zap.Error(err)) - c.Error("could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) - return - } - filtered := filterHeaders(a.log, &c.Request.Header) - attributes := make([]*object.Attribute, 0, len(filtered)) - // prepares attributes from filtered headers - for key, val := range filtered { - attribute := object.NewAttribute() - attribute.SetKey(key) - attribute.SetValue(val) - attributes = append(attributes, attribute) - } - // sets FileName attribute if it wasn't set from header - if _, ok := filtered[object.AttributeFileName]; !ok { - filename := object.NewAttribute() - filename.SetKey(object.AttributeFileName) - filename.SetValue(file.FileName()) - attributes = append(attributes, filename) - } - // sets Timestamp attribute if it wasn't set from header and enabled by settings - if _, ok := filtered[object.AttributeTimestamp]; !ok && a.enableDefaultTimestamp { - timestamp := object.NewAttribute() - timestamp.SetKey(object.AttributeTimestamp) - timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) - attributes = append(attributes, timestamp) - } - oid, bt := a.fetchOwnerAndBearerToken(c) - // prepares new object and fill it - raw := object.NewRaw() - raw.SetContainerID(cid) - raw.SetOwnerID(oid) - raw.SetAttributes(attributes...) - // tries to put file into NeoFS or throw error - // if addr, err = a.plant.Object().Put(c, raw.Object(), sdk.WithPutReader(file)); err != nil { - // TODO: Take this from a sync pool. - putOpts := new(neofs.PutOptions) - putOpts.Client, putOpts.SessionToken, err = a.plant.GetReusableArtifacts(c) - if err != nil { - log.Error("failed to get neofs client's reusable artifacts", zap.Error(err)) - c.Error("failed to get neofs client's reusable artifacts", fasthttp.StatusInternalServerError) - return - } - putOpts.BearerToken = bt - putOpts.ContainerID = cid - putOpts.OwnerID = oid - putOpts.PrepareObjectOnsite = false - putOpts.Reader = file - if addr, err = a.plant.Object().Put(c, putOpts); err != nil { - log.Error("could not store file in NeoFS", zap.Error(err)) - c.Error("could not store file in NeoFS", fasthttp.StatusBadRequest) - return - } - // tries to return response, otherwise, if something went wrong throw error - if err = newPutResponse(addr).encode(c); err != nil { - log.Error("could not prepare response", zap.Error(err)) - c.Error("could not prepare response", fasthttp.StatusBadRequest) - - return - } - // reports status code and content type - c.Response.SetStatusCode(fasthttp.StatusOK) - c.Response.Header.SetContentType(jsonHeader) -} From 3a5d9fe94c734276707c65f9ae6343498f2c4454 Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Wed, 31 Mar 2021 21:24:41 +0300 Subject: [PATCH 115/548] [#19] Extract downloading logic into a separate package Signed-off-by: Pavel Korotkov --- app.go | 39 ++++++--------- receive.go => downloader/download.go | 74 +++++++++++++++++++++------- uploader/upload.go | 20 +++++--- 3 files changed, 83 insertions(+), 50 deletions(-) rename receive.go => downloader/download.go (74%) diff --git a/app.go b/app.go index f1f1336..06b0bf7 100644 --- a/app.go +++ b/app.go @@ -6,8 +6,7 @@ import ( "strconv" "github.com/fasthttp/router" - "github.com/nspcc-dev/neofs-api-go/pkg/client" - "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-http-gate/downloader" "github.com/nspcc-dev/neofs-http-gate/logger" "github.com/nspcc-dev/neofs-http-gate/neofs" "github.com/nspcc-dev/neofs-http-gate/uploader" @@ -19,14 +18,10 @@ import ( type ( app struct { - plant neofs.ClientPlant - getOperations struct { - client client.Client - sessionToken *token.SessionToken - } log *zap.Logger + plant neofs.ClientPlant cfg *viper.Viper - wlog logger.Logger + auxiliaryLog logger.Logger web *fasthttp.Server jobDone chan struct{} webDone chan struct{} @@ -62,24 +57,20 @@ func WithConfig(c *viper.Viper) Option { func newApp(ctx context.Context, opt ...Option) App { a := &app{ - log: zap.L(), - cfg: viper.GetViper(), - web: new(fasthttp.Server), - + log: zap.L(), + cfg: viper.GetViper(), + web: new(fasthttp.Server), jobDone: make(chan struct{}), webDone: make(chan struct{}), } - for i := range opt { opt[i](a) } - a.enableDefaultTimestamp = a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) - - a.wlog = logger.GRPC(a.log) + a.auxiliaryLog = logger.GRPC(a.log) if a.cfg.GetBool(cmdVerbose) { - grpclog.SetLoggerV2(a.wlog) + grpclog.SetLoggerV2(a.auxiliaryLog) } // conTimeout := a.cfg.GetDuration(cfgConTimeout) @@ -124,10 +115,6 @@ func newApp(ctx context.Context, opt ...Option) App { if err != nil { a.log.Fatal("failed to create neofs client") } - a.getOperations.client, a.getOperations.sessionToken, err = a.plant.GetReusableArtifacts(ctx) - if err != nil { - a.log.Fatal("failed to get neofs client's reusable artifacts") - } return a } @@ -153,21 +140,25 @@ func (a *app) Serve(ctx context.Context) { close(a.webDone) }() uploader := uploader.New(a.log, a.plant, a.enableDefaultTimestamp) + downloader, err := downloader.New(ctx, a.log, a.plant) + if err != nil { + a.log.Fatal("failed to create downloader", zap.Error(err)) + } // Configure router. r := router.New() r.RedirectTrailingSlash = true r.POST("/upload/{cid}", uploader.Upload) a.log.Info("added path /upload/{cid}") - r.GET("/get/{cid}/{oid}", a.byAddress) + r.GET("/get/{cid}/{oid}", downloader.DownloadByAddress) a.log.Info("added path /get/{cid}/{oid}") - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.byAttribute) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", downloader.DownloadByAttribute) a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") // attaching /-/(ready,healthy) // attachHealthy(r, a.pool.Status) // enable metrics if a.cfg.GetBool(cmdMetrics) { a.log.Info("added path /metrics/") - attachMetrics(r, a.wlog) + attachMetrics(r, a.auxiliaryLog) } // enable pprof if a.cfg.GetBool(cmdPprof) { diff --git a/receive.go b/downloader/download.go similarity index 74% rename from receive.go rename to downloader/download.go index 25199f8..37da819 100644 --- a/receive.go +++ b/downloader/download.go @@ -1,6 +1,7 @@ -package main +package downloader import ( + "context" "io" "net/http" "path" @@ -9,8 +10,10 @@ import ( "sync" "time" + "github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-http-gate/neofs" "github.com/nspcc-dev/neofs-http-gate/tokens" "github.com/pkg/errors" @@ -20,6 +23,20 @@ import ( "google.golang.org/grpc/status" ) +var ( + getOptionsPool = sync.Pool{ + New: func() interface{} { + return new(neofs.GetOptions) + }, + } + + searchOptionsPool = sync.Pool{ + New: func() interface{} { + return new(neofs.SearchOptions) + }, + } +) + type ( detector struct { io.Writer @@ -29,8 +46,8 @@ type ( request struct { *fasthttp.RequestCtx - log *zap.Logger - obj neofs.ObjectClient + log *zap.Logger + objectClient neofs.ObjectClient } objectIDs []*object.ID @@ -61,7 +78,7 @@ func (r *request) receiveFile(options *neofs.GetOptions) { } writer := newDetector(r.Response.BodyWriter()) options.Writer = writer - obj, err := r.obj.Get(r.RequestCtx, options) + obj, err := r.objectClient.Get(r.RequestCtx, options) if err != nil { r.log.Error( "could not receive object", @@ -120,15 +137,34 @@ func (o objectIDs) Slice() []string { return res } -func (a *app) request(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { - return &request{ - RequestCtx: ctx, - log: log, - obj: a.plant.Object(), +type Downloader struct { + log *zap.Logger + plant neofs.ClientPlant + getOperations struct { + client client.Client + sessionToken *token.SessionToken } } -func (a *app) byAddress(c *fasthttp.RequestCtx) { +func New(ctx context.Context, log *zap.Logger, plant neofs.ClientPlant) (*Downloader, error) { + var err error + d := &Downloader{log: log, plant: plant} + d.getOperations.client, d.getOperations.sessionToken, err = d.plant.GetReusableArtifacts(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to get neofs client's reusable artifacts") + } + return d, nil +} + +func (a *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { + return &request{ + RequestCtx: ctx, + log: log, + objectClient: a.plant.Object(), + } +} + +func (a *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { var ( err error address = object.NewAddress() @@ -142,16 +178,16 @@ func (a *app) byAddress(c *fasthttp.RequestCtx) { c.Error("wrong object address", fasthttp.StatusBadRequest) return } - // TODO: Take this from a sync-pool. - getOpts := new(neofs.GetOptions) + getOpts := getOptionsPool.Get().(*neofs.GetOptions) + defer getOptionsPool.Put(getOpts) getOpts.Client = a.getOperations.client getOpts.SessionToken = a.getOperations.sessionToken getOpts.ObjectAddress = address getOpts.Writer = nil - a.request(c, log).receiveFile(getOpts) + a.newRequest(c, log).receiveFile(getOpts) } -func (a *app) byAttribute(c *fasthttp.RequestCtx) { +func (a *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { var ( err error scid, _ = c.UserValue("cid").(string) @@ -165,8 +201,8 @@ func (a *app) byAttribute(c *fasthttp.RequestCtx) { c.Error("wrong container id", fasthttp.StatusBadRequest) return } - // TODO: Take this from a sync-pool. - searchOpts := new(neofs.SearchOptions) + searchOpts := searchOptionsPool.Get().(*neofs.SearchOptions) + defer searchOptionsPool.Put(searchOpts) searchOpts.Client = a.getOperations.client searchOpts.SessionToken = a.getOperations.sessionToken searchOpts.BearerToken = nil @@ -191,11 +227,11 @@ func (a *app) byAttribute(c *fasthttp.RequestCtx) { address := object.NewAddress() address.SetContainerID(cid) address.SetObjectID(ids[0]) - // TODO: Take this from a sync-pool. - getOpts := new(neofs.GetOptions) + getOpts := getOptionsPool.Get().(*neofs.GetOptions) + defer getOptionsPool.Put(getOpts) getOpts.Client = a.getOperations.client getOpts.SessionToken = a.getOperations.sessionToken getOpts.ObjectAddress = address getOpts.Writer = nil - a.request(c, log).receiveFile(getOpts) + a.newRequest(c, log).receiveFile(getOpts) } diff --git a/uploader/upload.go b/uploader/upload.go index ad03b0d..a77272d 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -5,6 +5,7 @@ import ( "encoding/json" "io" "strconv" + "sync" "time" "github.com/nspcc-dev/neofs-api-go/pkg/container" @@ -17,6 +18,12 @@ import ( "go.uber.org/zap" ) +var putOptionsPool = sync.Pool{ + New: func() interface{} { + return new(neofs.PutOptions) + }, +} + type Uploader struct { log *zap.Logger plant neofs.ClientPlant @@ -89,15 +96,14 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { attributes = append(attributes, timestamp) } oid, bt := u.fetchOwnerAndBearerToken(c) - // prepares new object and fill it + // Prepare a new object and fill it. raw := object.NewRaw() raw.SetContainerID(cid) raw.SetOwnerID(oid) raw.SetAttributes(attributes...) - // tries to put file into NeoFS or throw error - // if addr, err = a.plant.Object().Put(c, raw.Object(), sdk.WithPutReader(file)); err != nil { - // TODO: Take this from a sync pool. - putOpts := new(neofs.PutOptions) + putOpts := putOptionsPool.Get().(*neofs.PutOptions) + defer putOptionsPool.Put(putOpts) + // Try to put file into NeoFS or throw an error. putOpts.Client, putOpts.SessionToken, err = u.plant.GetReusableArtifacts(c) if err != nil { log.Error("failed to get neofs client's reusable artifacts", zap.Error(err)) @@ -114,14 +120,14 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { c.Error("could not store file in NeoFS", fasthttp.StatusBadRequest) return } - // tries to return response, otherwise, if something went wrong throw error + // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { log.Error("could not prepare response", zap.Error(err)) c.Error("could not prepare response", fasthttp.StatusBadRequest) return } - // reports status code and content type + // Report status code and content type. c.Response.SetStatusCode(fasthttp.StatusOK) c.Response.Header.SetContentType(jsonHeader) } From 0b6e138270379d6ff7268141a2951eb99d48d06d Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Wed, 31 Mar 2021 22:08:39 +0300 Subject: [PATCH 116/548] [#19] Prepare using connection pool Signed-off-by: Pavel Korotkov --- app.go | 86 +++++++++++++++++-------------------------- neofs/client-plant.go | 20 +++++++++- neofs/credentials.go | 22 +++++------ 3 files changed, 64 insertions(+), 64 deletions(-) diff --git a/app.go b/app.go index 06b0bf7..cd7c9b9 100644 --- a/app.go +++ b/app.go @@ -18,14 +18,13 @@ import ( type ( app struct { - log *zap.Logger - plant neofs.ClientPlant - cfg *viper.Viper - auxiliaryLog logger.Logger - web *fasthttp.Server - jobDone chan struct{} - webDone chan struct{} - enableDefaultTimestamp bool + log *zap.Logger + plant neofs.ClientPlant + cfg *viper.Viper + auxiliaryLog logger.Logger + webServer *fasthttp.Server + jobDone chan struct{} + webDone chan struct{} } App interface { @@ -57,61 +56,55 @@ func WithConfig(c *viper.Viper) Option { func newApp(ctx context.Context, opt ...Option) App { a := &app{ - log: zap.L(), - cfg: viper.GetViper(), - web: new(fasthttp.Server), - jobDone: make(chan struct{}), - webDone: make(chan struct{}), + log: zap.L(), + cfg: viper.GetViper(), + webServer: new(fasthttp.Server), + jobDone: make(chan struct{}), + webDone: make(chan struct{}), } for i := range opt { opt[i](a) } - a.enableDefaultTimestamp = a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) a.auxiliaryLog = logger.GRPC(a.log) - if a.cfg.GetBool(cmdVerbose) { grpclog.SetLoggerV2(a.auxiliaryLog) } - // conTimeout := a.cfg.GetDuration(cfgConTimeout) // reqTimeout := a.cfg.GetDuration(cfgReqTimeout) // tckTimeout := a.cfg.GetDuration(cfgRebalance) - // -- setup FastHTTP server -- - a.web.Name = "neofs-http-gate" - a.web.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) - a.web.WriteBufferSize = a.cfg.GetInt(cfgWebWriteBufferSize) - a.web.ReadTimeout = a.cfg.GetDuration(cfgWebReadTimeout) - a.web.WriteTimeout = a.cfg.GetDuration(cfgWebWriteTimeout) - a.web.DisableHeaderNamesNormalizing = true - a.web.NoDefaultServerHeader = true - a.web.NoDefaultContentType = true - a.web.MaxRequestBodySize = a.cfg.GetInt(cfgWebMaxRequestBodySize) - + a.webServer.Name = "neofs-http-gate" + a.webServer.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) + a.webServer.WriteBufferSize = a.cfg.GetInt(cfgWebWriteBufferSize) + a.webServer.ReadTimeout = a.cfg.GetDuration(cfgWebReadTimeout) + a.webServer.WriteTimeout = a.cfg.GetDuration(cfgWebWriteTimeout) + a.webServer.DisableHeaderNamesNormalizing = true + a.webServer.NoDefaultServerHeader = true + a.webServer.NoDefaultContentType = true + a.webServer.MaxRequestBodySize = a.cfg.GetInt(cfgWebMaxRequestBodySize) // -- -- -- -- -- -- FIXME -- -- -- -- -- -- // Does not work with StreamRequestBody, // some bugs with readMultipartForm // https://github.com/valyala/fasthttp/issues/968 - a.web.DisablePreParseMultipartForm = true - a.web.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) + a.webServer.DisablePreParseMultipartForm = true + a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - - var cl connectionList + var cl neofs.ConnectionList for i := 0; ; i++ { address := a.cfg.GetString(cfgPeers + "." + strconv.Itoa(i) + ".address") weight := a.cfg.GetFloat64(cfgPeers + "." + strconv.Itoa(i) + ".weight") if address == "" { break } - cl = append(cl, connection{address: address, weight: weight}) - a.log.Info("add connection peer", zap.String("address", address), zap.Float64("weight", weight)) + cl.Add(address, weight) + a.log.Info("add connection", zap.String("address", address), zap.Float64("weight", weight)) } sort.Sort(sort.Reverse(cl)) - cred, err := neofs.NewCredentials(a.cfg.GetString(cmdNeoFSKey)) + creds, err := neofs.NewCredentials(a.cfg.GetString(cmdNeoFSKey)) if err != nil { - a.log.Fatal("could not get credentials", zap.Error(err)) + a.log.Fatal("could not get neofs credentials", zap.Error(err)) } - a.plant, err = neofs.NewClientPlant(ctx, cl[0].address, cred) + a.plant, err = neofs.NewClientPlant(ctx, cl, creds) if err != nil { a.log.Fatal("failed to create neofs client") } @@ -120,7 +113,6 @@ func newApp(ctx context.Context, opt ...Option) App { func (a *app) Wait() { a.log.Info("starting application") - select { case <-a.jobDone: // wait for job is stopped <-a.webDone @@ -136,10 +128,11 @@ func (a *app) Worker(ctx context.Context) { func (a *app) Serve(ctx context.Context) { go func() { <-ctx.Done() - a.log.Info("shutting down web server", zap.Error(a.web.Shutdown())) + a.log.Info("shutting down web server", zap.Error(a.webServer.Shutdown())) close(a.webDone) }() - uploader := uploader.New(a.log, a.plant, a.enableDefaultTimestamp) + edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) + uploader := uploader.New(a.log, a.plant, edts) downloader, err := downloader.New(ctx, a.log, a.plant) if err != nil { a.log.Fatal("failed to create downloader", zap.Error(err)) @@ -167,19 +160,8 @@ func (a *app) Serve(ctx context.Context) { } bind := a.cfg.GetString(cfgListenAddress) a.log.Info("running web server", zap.String("address", bind)) - a.web.Handler = r.Handler - if err := a.web.ListenAndServe(bind); err != nil { + a.webServer.Handler = r.Handler + if err := a.webServer.ListenAndServe(bind); err != nil { a.log.Fatal("could not start server", zap.Error(err)) } } - -type connection struct { - address string - weight float64 -} - -type connectionList []connection - -func (p connectionList) Len() int { return len(p) } -func (p connectionList) Less(i, j int) bool { return p[i].weight < p[j].weight } -func (p connectionList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/neofs/client-plant.go b/neofs/client-plant.go index cf3aaa9..babcaff 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -102,9 +102,27 @@ func (cc *neofsClient) OwnerID() *owner.ID { return cc.ownerID } -func NewClientPlant(ctx context.Context, address string, creds Credentials) (ClientPlant, error) { +type Connection struct { + address string + weight float64 +} + +type ConnectionList []Connection + +func (p ConnectionList) Len() int { return len(p) } +func (p ConnectionList) Less(i, j int) bool { return p[i].weight < p[j].weight } +func (p ConnectionList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +func (cl *ConnectionList) Add(address string, weight float64) ConnectionList { + *cl = append(*cl, Connection{address, weight}) + return *cl +} + +func NewClientPlant(ctx context.Context, connectionList ConnectionList, creds Credentials) (ClientPlant, error) { toctx, c := context.WithTimeout(ctx, nodeConnectionTimeout) defer c() + // TODO: Use connection pool here. + address := connectionList[0].address conn, err := grpc.DialContext(toctx, address, grpc.WithInsecure(), grpc.WithBlock()) if err != nil { if err == context.DeadlineExceeded { diff --git a/neofs/credentials.go b/neofs/credentials.go index 48faf64..20b8c9b 100644 --- a/neofs/credentials.go +++ b/neofs/credentials.go @@ -16,10 +16,10 @@ type ( PrivateKey() *ecdsa.PrivateKey } - cred struct { - key *ecdsa.PrivateKey - owner *owner.ID - wif string + credentials struct { + key *ecdsa.PrivateKey + ownerID *owner.ID + wif string } ) @@ -34,26 +34,26 @@ func NewCredentials(secret string) (Credentials, error) { } // PrivateKey returns ecdsa.PrivateKey. -func (c *cred) PrivateKey() *ecdsa.PrivateKey { +func (c *credentials) PrivateKey() *ecdsa.PrivateKey { return c.key } // PublicKey returns ecdsa.PublicKey. -func (c *cred) PublicKey() *ecdsa.PublicKey { +func (c *credentials) PublicKey() *ecdsa.PublicKey { return &c.key.PublicKey } // Owner returns owner.ID. -func (c *cred) Owner() *owner.ID { - return c.owner +func (c *credentials) Owner() *owner.ID { + return c.ownerID } // WIF returns string representation of WIF. -func (c *cred) WIF() string { +func (c *credentials) WIF() string { return c.wif } -func setFromPrivateKey(key *ecdsa.PrivateKey) (*cred, error) { +func setFromPrivateKey(key *ecdsa.PrivateKey) (*credentials, error) { wallet, err := owner.NEO3WalletFromPublicKey(&key.PublicKey) if err != nil { return nil, err @@ -63,5 +63,5 @@ func setFromPrivateKey(key *ecdsa.PrivateKey) (*cred, error) { if err != nil { return nil, err } - return &cred{key: key, owner: ownerID, wif: wif}, nil + return &credentials{key: key, ownerID: ownerID, wif: wif}, nil } From aedd468c1842e7eb8f057bd7d6724293ae0b2a34 Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Wed, 31 Mar 2021 22:15:10 +0300 Subject: [PATCH 117/548] [#19] Move connection list sorting to a proper place Signed-off-by: Pavel Korotkov --- app.go | 2 -- neofs/client-plant.go | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app.go b/app.go index cd7c9b9..1b0f981 100644 --- a/app.go +++ b/app.go @@ -2,7 +2,6 @@ package main import ( "context" - "sort" "strconv" "github.com/fasthttp/router" @@ -99,7 +98,6 @@ func newApp(ctx context.Context, opt ...Option) App { cl.Add(address, weight) a.log.Info("add connection", zap.String("address", address), zap.Float64("weight", weight)) } - sort.Sort(sort.Reverse(cl)) creds, err := neofs.NewCredentials(a.cfg.GetString(cmdNeoFSKey)) if err != nil { a.log.Fatal("could not get neofs credentials", zap.Error(err)) diff --git a/neofs/client-plant.go b/neofs/client-plant.go index babcaff..7ce4824 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -6,6 +6,7 @@ import ( "crypto/ecdsa" "io" "math" + "sort" "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" @@ -121,6 +122,7 @@ func (cl *ConnectionList) Add(address string, weight float64) ConnectionList { func NewClientPlant(ctx context.Context, connectionList ConnectionList, creds Credentials) (ClientPlant, error) { toctx, c := context.WithTimeout(ctx, nodeConnectionTimeout) defer c() + sort.Sort(sort.Reverse(connectionList)) // TODO: Use connection pool here. address := connectionList[0].address conn, err := grpc.DialContext(toctx, address, grpc.WithInsecure(), grpc.WithBlock()) From c909c99f726c48228825c4db9c1ee4019bdb9730 Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Thu, 1 Apr 2021 09:51:06 +0300 Subject: [PATCH 118/548] [#19] Rename client-oriented entities before adding fake/mock ones Signed-off-by: Pavel Korotkov --- neofs/client-plant.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/neofs/client-plant.go b/neofs/client-plant.go index 7ce4824..39c7629 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -72,18 +72,18 @@ type ClientPlant interface { OwnerID() *owner.ID } -type objectClient struct { +type neofsObjectClient struct { key *ecdsa.PrivateKey conn *grpc.ClientConn } -type neofsClient struct { +type neofsClientPlant struct { key *ecdsa.PrivateKey ownerID *owner.ID conn *grpc.ClientConn } -func (cc *neofsClient) GetReusableArtifacts(ctx context.Context) (client.Client, *token.SessionToken, error) { +func (cc *neofsClientPlant) GetReusableArtifacts(ctx context.Context) (client.Client, *token.SessionToken, error) { c, err := client.New(client.WithDefaultPrivateKey(cc.key), client.WithGRPCConnection(cc.conn)) if err != nil { return nil, nil, errors.Wrap(err, "failed to create reusable neofs client") @@ -95,11 +95,11 @@ func (cc *neofsClient) GetReusableArtifacts(ctx context.Context) (client.Client, return c, st, nil } -func (cc *neofsClient) Object() ObjectClient { - return &objectClient{key: cc.key, conn: cc.conn} +func (cc *neofsClientPlant) Object() ObjectClient { + return &neofsObjectClient{key: cc.key, conn: cc.conn} } -func (cc *neofsClient) OwnerID() *owner.ID { +func (cc *neofsClientPlant) OwnerID() *owner.ID { return cc.ownerID } @@ -132,14 +132,14 @@ func NewClientPlant(ctx context.Context, connectionList ConnectionList, creds Cr } return nil, err } - return &neofsClient{ + return &neofsClientPlant{ key: creds.PrivateKey(), ownerID: creds.Owner(), conn: conn, }, nil } -func (oc *objectClient) Put(ctx context.Context, options *PutOptions) (*object.Address, error) { +func (oc *neofsObjectClient) Put(ctx context.Context, options *PutOptions) (*object.Address, error) { var ( err error objectID *object.ID @@ -199,7 +199,7 @@ func (oc *objectClient) Put(ctx context.Context, options *PutOptions) (*object.A return address, nil } -func (oc *objectClient) Get(ctx context.Context, options *GetOptions) (*object.Object, error) { +func (oc *neofsObjectClient) Get(ctx context.Context, options *GetOptions) (*object.Object, error) { var ( err error obj *object.Object @@ -216,7 +216,7 @@ func (oc *objectClient) Get(ctx context.Context, options *GetOptions) (*object.O return obj, err } -func (oc *objectClient) Search(ctx context.Context, options *SearchOptions) ([]*object.ID, error) { +func (oc *neofsObjectClient) Search(ctx context.Context, options *SearchOptions) ([]*object.ID, error) { sfs := object.NewSearchFilters() sfs.AddRootFilter() sfs.AddFilter(options.Attribute.Key, options.Attribute.Value, object.MatchStringEqual) @@ -231,7 +231,7 @@ func (oc *objectClient) Search(ctx context.Context, options *SearchOptions) ([]* ) } -func (oc *objectClient) Delete(ctx context.Context, options *DeleteOptions) error { +func (oc *neofsObjectClient) Delete(ctx context.Context, options *DeleteOptions) error { ops := new(client.DeleteObjectParams).WithAddress(options.ObjectAddress) err := options.Client.DeleteObject( ctx, From 62a03251ced5eb70c239f9db02cd2db1b74e390c Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Mon, 5 Apr 2021 17:48:01 +0300 Subject: [PATCH 119/548] Add connection pool implementation (part 1) Signed-off-by: Pavel Korotkov --- .test.env | 6 ++- app.go | 35 +++++++++------- connections/generator.go | 76 +++++++++++++++++++++++++++++++++++ connections/pool.go | 86 ++++++++++++++++++++++++++++++++++++++++ health.go => health._go_ | 0 neofs/client-plant.go | 61 ++++++---------------------- pool/pool.go | 39 ------------------ settings.go | 2 +- uploader/upload.go | 12 +++--- 9 files changed, 205 insertions(+), 112 deletions(-) create mode 100644 connections/generator.go create mode 100644 connections/pool.go rename health.go => health._go_ (100%) delete mode 100644 pool/pool.go diff --git a/.test.env b/.test.env index 995d4a3..840ef5b 100644 --- a/.test.env +++ b/.test.env @@ -9,5 +9,7 @@ HTTP_GW_KEEPALIVE_TIMEOUT=300s HTTP_GW_KEEPALIVE_TIME=120s HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=True HTTP_GW_CONN_TTL=1h -HTTP_GW_PEERS_0_WEIGHT=1.0 -HTTP_GW_PEERS_0_ADDRESS=s01.neofs.devenv:8080 \ No newline at end of file +HTTP_GW_PEERS_0_WEIGHT=0.6 +HTTP_GW_PEERS_0_ADDRESS=s01.neofs.devenv:8080 +HTTP_GW_PEERS_1_WEIGHT=0.4 +HTTP_GW_PEERS_1_ADDRESS=s02.neofs.devenv:8080 \ No newline at end of file diff --git a/app.go b/app.go index 1b0f981..bbfead4 100644 --- a/app.go +++ b/app.go @@ -5,6 +5,7 @@ import ( "strconv" "github.com/fasthttp/router" + "github.com/nspcc-dev/neofs-http-gate/connections" "github.com/nspcc-dev/neofs-http-gate/downloader" "github.com/nspcc-dev/neofs-http-gate/logger" "github.com/nspcc-dev/neofs-http-gate/neofs" @@ -68,9 +69,6 @@ func newApp(ctx context.Context, opt ...Option) App { if a.cfg.GetBool(cmdVerbose) { grpclog.SetLoggerV2(a.auxiliaryLog) } - // conTimeout := a.cfg.GetDuration(cfgConTimeout) - // reqTimeout := a.cfg.GetDuration(cfgReqTimeout) - // tckTimeout := a.cfg.GetDuration(cfgRebalance) // -- setup FastHTTP server -- a.webServer.Name = "neofs-http-gate" a.webServer.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) @@ -82,29 +80,38 @@ func newApp(ctx context.Context, opt ...Option) App { a.webServer.NoDefaultContentType = true a.webServer.MaxRequestBodySize = a.cfg.GetInt(cfgWebMaxRequestBodySize) // -- -- -- -- -- -- FIXME -- -- -- -- -- -- - // Does not work with StreamRequestBody, - // some bugs with readMultipartForm - // https://github.com/valyala/fasthttp/issues/968 + // Does not work with StreamRequestBody due to bugs with + // readMultipartForm, see https://github.com/valyala/fasthttp/issues/968 a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - var cl neofs.ConnectionList + creds, err := neofs.NewCredentials(a.cfg.GetString(cmdNeoFSKey)) + if err != nil { + a.log.Fatal("failed to get neofs credentials", zap.Error(err)) + } + pb := new(connections.PoolBuilder) for i := 0; ; i++ { address := a.cfg.GetString(cfgPeers + "." + strconv.Itoa(i) + ".address") weight := a.cfg.GetFloat64(cfgPeers + "." + strconv.Itoa(i) + ".weight") if address == "" { break } - cl.Add(address, weight) + pb.AddNode(address, weight) a.log.Info("add connection", zap.String("address", address), zap.Float64("weight", weight)) } - creds, err := neofs.NewCredentials(a.cfg.GetString(cmdNeoFSKey)) - if err != nil { - a.log.Fatal("could not get neofs credentials", zap.Error(err)) + opts := &connections.PoolBuilderOptions{ + Key: creds.PrivateKey(), + NodeConnectionTimeout: a.cfg.GetDuration(cfgConTimeout), + NodeRequestTimeout: a.cfg.GetDuration(cfgReqTimeout), + ClientRebalanceInterval: a.cfg.GetDuration(cfgRebalance), } - a.plant, err = neofs.NewClientPlant(ctx, cl, creds) + pool, err := pb.Build(ctx, opts) if err != nil { - a.log.Fatal("failed to create neofs client") + a.log.Fatal("failed to create connection pool", zap.Error(err)) + } + a.plant, err = neofs.NewClientPlant(ctx, pool, creds) + if err != nil { + a.log.Fatal("failed to create neofs client plant") } return a } @@ -144,8 +151,6 @@ func (a *app) Serve(ctx context.Context) { a.log.Info("added path /get/{cid}/{oid}") r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", downloader.DownloadByAttribute) a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") - // attaching /-/(ready,healthy) - // attachHealthy(r, a.pool.Status) // enable metrics if a.cfg.GetBool(cmdMetrics) { a.log.Info("added path /metrics/") diff --git a/connections/generator.go b/connections/generator.go new file mode 100644 index 0000000..a03bd26 --- /dev/null +++ b/connections/generator.go @@ -0,0 +1,76 @@ +package connections + +import "math/rand" + +// https://www.keithschwarz.com/darts-dice-coins/ +type Generator struct { + randomGenerator *rand.Rand + probabilities []float64 + alias []int +} + +type workList []int + +func (wl *workList) push(e int) { + *wl = append(*wl, e) +} + +func (wl *workList) pop() int { + l := len(*wl) - 1 + n := (*wl)[l] + *wl = (*wl)[:l] + return n +} + +func NewGenerator(probabilities []float64, source rand.Source) *Generator { + generator := &Generator{} + var ( + small workList + large workList + ) + n := len(probabilities) + generator.randomGenerator = rand.New(source) + generator.probabilities = make([]float64, n) + generator.alias = make([]int, n) + // Compute scaled probabilities. + p := make([]float64, n) + for i := 0; i < n; i++ { + p[i] = probabilities[i] * float64(n) + } + for i, pi := range p { + if pi < 1 { + small = append(small, i) + } else { + large = append(large, i) + } + } + for len(large) > 0 && len(small) > 0 { + l, g := small.pop(), large.pop() + generator.probabilities[l] = p[l] + generator.alias[l] = g + p[g] = (p[g] + p[l]) - 1 + if p[g] < 1 { + small.push(g) + } else { + large.push(g) + } + } + for len(large) > 0 { + g := large.pop() + generator.probabilities[g] = 1 + } + for len(small) > 0 { + l := small.pop() + generator.probabilities[l] = 1 + } + return generator +} + +func (g *Generator) Next() int { + n := len(g.alias) + i := g.randomGenerator.Intn(n) + if g.randomGenerator.Float64() < g.probabilities[i] { + return i + } + return g.alias[i] +} diff --git a/connections/pool.go b/connections/pool.go new file mode 100644 index 0000000..5fefbf0 --- /dev/null +++ b/connections/pool.go @@ -0,0 +1,86 @@ +package connections + +import ( + "context" + "crypto/ecdsa" + "errors" + "math" + "math/rand" + "time" + + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "google.golang.org/grpc" +) + +type PoolBuilderOptions struct { + Key *ecdsa.PrivateKey + NodeConnectionTimeout time.Duration + NodeRequestTimeout time.Duration + ClientRebalanceInterval time.Duration +} + +type PoolBuilder struct { + addresses []string + weights []float64 +} + +func (pb *PoolBuilder) AddNode(address string, weight float64) *PoolBuilder { + pb.addresses = append(pb.addresses, address) + pb.weights = append(pb.weights, weight) + return pb +} + +func (pb *PoolBuilder) Build(ctx context.Context, options *PoolBuilderOptions) (Pool, error) { + totalWeight := 0.0 + for _, w := range pb.weights { + totalWeight += w + } + if math.Abs(totalWeight-1.0) >= 1e-4 { + return nil, errors.New("total weight must be equal to unity") + } + var cons = make([]*grpc.ClientConn, len(pb.addresses)) + for i, address := range pb.addresses { + con, err := func() (*grpc.ClientConn, error) { + toctx, c := context.WithTimeout(ctx, options.NodeConnectionTimeout) + defer c() + return grpc.DialContext(toctx, address, grpc.WithInsecure(), grpc.WithBlock()) + }() + if err != nil { + return nil, err + } + cons[i] = con + } + return new(pb.weights, options.Key, cons) +} + +type Pool interface { + Client() client.Client +} + +type pool struct { + generator *Generator + clients []client.Client +} + +func new(weights []float64, key *ecdsa.PrivateKey, connections []*grpc.ClientConn) (Pool, error) { + clients := make([]client.Client, len(weights)) + for i, con := range connections { + c, err := client.New(client.WithDefaultPrivateKey(key), client.WithGRPCConnection(con)) + if err != nil { + return nil, err + } + clients[i] = c + } + source := rand.NewSource(time.Now().UnixNano()) + return &pool{ + generator: NewGenerator(weights, source), + clients: clients, + }, nil +} + +func (p *pool) Client() client.Client { + if len(p.clients) == 1 { + return p.clients[0] + } + return p.clients[p.generator.Next()] +} diff --git a/health.go b/health._go_ similarity index 100% rename from health.go rename to health._go_ diff --git a/neofs/client-plant.go b/neofs/client-plant.go index 39c7629..0078008 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -6,24 +6,19 @@ import ( "crypto/ecdsa" "io" "math" - "sort" - "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-http-gate/connections" objectCore "github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/transformer" "github.com/pkg/errors" - "google.golang.org/grpc" ) -const ( - nodeConnectionTimeout = 10 * time.Second - maxObjectSize = uint64(1 << 26) // 64MiB -) +const maxObjectSize = uint64(1 << 28) // Limit objects to 256 MiB. type BaseOptions struct { Client client.Client @@ -74,20 +69,17 @@ type ClientPlant interface { type neofsObjectClient struct { key *ecdsa.PrivateKey - conn *grpc.ClientConn + pool connections.Pool } type neofsClientPlant struct { key *ecdsa.PrivateKey ownerID *owner.ID - conn *grpc.ClientConn + pool connections.Pool } -func (cc *neofsClientPlant) GetReusableArtifacts(ctx context.Context) (client.Client, *token.SessionToken, error) { - c, err := client.New(client.WithDefaultPrivateKey(cc.key), client.WithGRPCConnection(cc.conn)) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to create reusable neofs client") - } +func (cp *neofsClientPlant) GetReusableArtifacts(ctx context.Context) (client.Client, *token.SessionToken, error) { + c := cp.pool.Client() st, err := c.CreateSession(ctx, math.MaxUint64) if err != nil { return nil, nil, errors.Wrap(err, "failed to create reusable neofs session token") @@ -96,47 +88,18 @@ func (cc *neofsClientPlant) GetReusableArtifacts(ctx context.Context) (client.Cl } func (cc *neofsClientPlant) Object() ObjectClient { - return &neofsObjectClient{key: cc.key, conn: cc.conn} + return &neofsObjectClient{ + key: cc.key, + pool: cc.pool, + } } func (cc *neofsClientPlant) OwnerID() *owner.ID { return cc.ownerID } -type Connection struct { - address string - weight float64 -} - -type ConnectionList []Connection - -func (p ConnectionList) Len() int { return len(p) } -func (p ConnectionList) Less(i, j int) bool { return p[i].weight < p[j].weight } -func (p ConnectionList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - -func (cl *ConnectionList) Add(address string, weight float64) ConnectionList { - *cl = append(*cl, Connection{address, weight}) - return *cl -} - -func NewClientPlant(ctx context.Context, connectionList ConnectionList, creds Credentials) (ClientPlant, error) { - toctx, c := context.WithTimeout(ctx, nodeConnectionTimeout) - defer c() - sort.Sort(sort.Reverse(connectionList)) - // TODO: Use connection pool here. - address := connectionList[0].address - conn, err := grpc.DialContext(toctx, address, grpc.WithInsecure(), grpc.WithBlock()) - if err != nil { - if err == context.DeadlineExceeded { - err = errors.New("failed to connect to neofs node") - } - return nil, err - } - return &neofsClientPlant{ - key: creds.PrivateKey(), - ownerID: creds.Owner(), - conn: conn, - }, nil +func NewClientPlant(ctx context.Context, pool connections.Pool, creds Credentials) (ClientPlant, error) { + return &neofsClientPlant{key: creds.PrivateKey(), ownerID: creds.Owner(), pool: pool}, nil } func (oc *neofsObjectClient) Put(ctx context.Context, options *PutOptions) (*object.Address, error) { diff --git a/pool/pool.go b/pool/pool.go deleted file mode 100644 index 94874ec..0000000 --- a/pool/pool.go +++ /dev/null @@ -1,39 +0,0 @@ -package pool - -import ( - "context" - - "github.com/nspcc-dev/neofs-api-go/pkg/token" - "google.golang.org/grpc" -) - -type Client interface { - // receive status of connection pool - Status() error - // worker should be run in goroutine to re-balancing - Worker(context.Context) - Connection(context.Context) (*grpc.ClientConn, error) - Session(context.Context, *grpc.ClientConn) (*token.SessionToken, error) -} - -type pool struct{} - -func (p *pool) Status() error { - return nil -} - -func (p *pool) Worker(ctx context.Context) { - panic("not implemented") -} - -func (p *pool) Connection(ctx context.Context) (*grpc.ClientConn, error) { - panic("not implemented") -} - -func (p *pool) Session(ctx context.Context, conn *grpc.ClientConn) (*token.SessionToken, error) { - panic("not implemented") -} - -func New() Client { - return &pool{} -} diff --git a/settings.go b/settings.go index 0e5f79f..7844293 100644 --- a/settings.go +++ b/settings.go @@ -115,7 +115,7 @@ func settings() *viper.Viper { peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") // set prefers: - v.Set(cfgApplicationName, "neofs-http-gw") + v.Set(cfgApplicationName, "neofs-http-gate") v.Set(cfgApplicationVersion, Version) // set defaults: diff --git a/uploader/upload.go b/uploader/upload.go index a77272d..158939c 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -18,6 +18,8 @@ import ( "go.uber.org/zap" ) +const jsonHeader = "application/json; charset=UTF-8" + var putOptionsPool = sync.Pool{ New: func() interface{} { return new(neofs.PutOptions) @@ -140,16 +142,14 @@ func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*owner.ID, *to } type putResponse struct { - OID string `json:"object_id"` - CID string `json:"container_id"` + ObjectID string `json:"object_id"` + ContainerID string `json:"container_id"` } -const jsonHeader = "application/json; charset=UTF-8" - func newPutResponse(addr *object.Address) *putResponse { return &putResponse{ - OID: addr.ObjectID().String(), - CID: addr.ContainerID().String(), + ObjectID: addr.ObjectID().String(), + ContainerID: addr.ContainerID().String(), } } From a44551d42b657949fa0dec58f44db26e423966d6 Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Mon, 5 Apr 2021 20:10:03 +0300 Subject: [PATCH 120/548] Add connection pool implementation (part 2) Signed-off-by: Pavel Korotkov --- .test.env | 4 +-- connections/generator.go | 8 ++--- connections/pool.go | 72 ++++++++++++++++++++++++++++++++-------- global/context.go | 7 ++-- health._go_ | 33 ------------------ neofs/client-plant.go | 3 ++ 6 files changed, 70 insertions(+), 57 deletions(-) delete mode 100644 health._go_ diff --git a/.test.env b/.test.env index 840ef5b..6eaea9e 100644 --- a/.test.env +++ b/.test.env @@ -9,7 +9,7 @@ HTTP_GW_KEEPALIVE_TIMEOUT=300s HTTP_GW_KEEPALIVE_TIME=120s HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=True HTTP_GW_CONN_TTL=1h -HTTP_GW_PEERS_0_WEIGHT=0.6 +HTTP_GW_PEERS_0_WEIGHT=0.4 HTTP_GW_PEERS_0_ADDRESS=s01.neofs.devenv:8080 -HTTP_GW_PEERS_1_WEIGHT=0.4 +HTTP_GW_PEERS_1_WEIGHT=0.6 HTTP_GW_PEERS_1_ADDRESS=s02.neofs.devenv:8080 \ No newline at end of file diff --git a/connections/generator.go b/connections/generator.go index a03bd26..435386a 100644 --- a/connections/generator.go +++ b/connections/generator.go @@ -3,7 +3,7 @@ package connections import "math/rand" // https://www.keithschwarz.com/darts-dice-coins/ -type Generator struct { +type Sampler struct { randomGenerator *rand.Rand probabilities []float64 alias []int @@ -22,8 +22,8 @@ func (wl *workList) pop() int { return n } -func NewGenerator(probabilities []float64, source rand.Source) *Generator { - generator := &Generator{} +func NewSampler(probabilities []float64, source rand.Source) *Sampler { + generator := &Sampler{} var ( small workList large workList @@ -66,7 +66,7 @@ func NewGenerator(probabilities []float64, source rand.Source) *Generator { return generator } -func (g *Generator) Next() int { +func (g *Sampler) Next() int { n := len(g.alias) i := g.randomGenerator.Intn(n) if g.randomGenerator.Float64() < g.probabilities[i] { diff --git a/connections/pool.go b/connections/pool.go index 5fefbf0..48c859b 100644 --- a/connections/pool.go +++ b/connections/pool.go @@ -6,6 +6,7 @@ import ( "errors" "math" "math/rand" + "sync" "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" @@ -17,6 +18,8 @@ type PoolBuilderOptions struct { NodeConnectionTimeout time.Duration NodeRequestTimeout time.Duration ClientRebalanceInterval time.Duration + weights []float64 + connections []*grpc.ClientConn } type PoolBuilder struct { @@ -50,7 +53,9 @@ func (pb *PoolBuilder) Build(ctx context.Context, options *PoolBuilderOptions) ( } cons[i] = con } - return new(pb.weights, options.Key, cons) + options.weights = pb.weights + options.connections = cons + return new(ctx, options) } type Pool interface { @@ -58,29 +63,70 @@ type Pool interface { } type pool struct { - generator *Generator - clients []client.Client + lock sync.RWMutex + sampler *Sampler + clients []client.Client + healthy []bool } -func new(weights []float64, key *ecdsa.PrivateKey, connections []*grpc.ClientConn) (Pool, error) { - clients := make([]client.Client, len(weights)) - for i, con := range connections { - c, err := client.New(client.WithDefaultPrivateKey(key), client.WithGRPCConnection(con)) +func new(ctx context.Context, options *PoolBuilderOptions) (Pool, error) { + n := len(options.weights) + clients := make([]client.Client, n) + healthy := make([]bool, n) + for i, con := range options.connections { + c, err := client.New(client.WithDefaultPrivateKey(options.Key), client.WithGRPCConnection(con)) if err != nil { return nil, err } clients[i] = c + healthy[i] = true } source := rand.NewSource(time.Now().UnixNano()) - return &pool{ - generator: NewGenerator(weights, source), - clients: clients, - }, nil + pool := &pool{ + sampler: NewSampler(options.weights, source), + clients: clients, + healthy: healthy, + } + go func() { + ticker := time.NewTimer(options.ClientRebalanceInterval) + for range ticker.C { + ok := true + for i, client := range pool.clients { + func() { + tctx, c := context.WithTimeout(ctx, options.NodeRequestTimeout) + defer c() + if _, err := client.EndpointInfo(tctx); err != nil { + ok = false + } + pool.lock.Lock() + pool.healthy[i] = ok + pool.lock.Unlock() + }() + } + ticker.Reset(options.ClientRebalanceInterval) + } + }() + return pool, nil } func (p *pool) Client() client.Client { + p.lock.RLock() + defer p.lock.RUnlock() if len(p.clients) == 1 { - return p.clients[0] + if p.healthy[0] { + return p.clients[0] + } + return nil } - return p.clients[p.generator.Next()] + var i *int = nil + for k := 0; k < 10; k++ { + i_ := p.sampler.Next() + if p.healthy[i_] { + i = &i_ + } + } + if i != nil { + return p.clients[*i] + } + return nil } diff --git a/global/context.go b/global/context.go index 19b3c53..0fd29cf 100644 --- a/global/context.go +++ b/global/context.go @@ -8,16 +8,13 @@ import ( ) var ( - globalContext context.Context - globalContextOnce sync.Once - globalContextBarrier = make(chan struct{}) + globalContext context.Context + globalContextOnce sync.Once ) func Context() context.Context { globalContextOnce.Do(func() { globalContext, _ = signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) - close(globalContextBarrier) }) - <-globalContextBarrier return globalContext } diff --git a/health._go_ b/health._go_ deleted file mode 100644 index 719667b..0000000 --- a/health._go_ +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "github.com/fasthttp/router" - "github.com/valyala/fasthttp" -) - -type stater func() error - -const ( - healthyState = "NeoFS HTTP Gateway is " - defaultContentType = "text/plain; charset=utf-8" -) - -func attachHealthy(r *router.Router, e stater) { - r.GET("/-/ready/", func(ctx *fasthttp.RequestCtx) { - ctx.SetStatusCode(fasthttp.StatusOK) - ctx.SetBodyString(healthyState + "ready") - }) - r.GET("/-/healthy/", func(c *fasthttp.RequestCtx) { - code := fasthttp.StatusOK - msg := "healthy" - - if err := e(); err != nil { - msg = "unhealthy: " + err.Error() - code = fasthttp.StatusBadRequest - } - c.Response.Reset() - c.SetStatusCode(code) - c.SetContentType(defaultContentType) - c.SetBodyString(healthyState + msg) - }) -} diff --git a/neofs/client-plant.go b/neofs/client-plant.go index 0078008..1430619 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -80,6 +80,9 @@ type neofsClientPlant struct { func (cp *neofsClientPlant) GetReusableArtifacts(ctx context.Context) (client.Client, *token.SessionToken, error) { c := cp.pool.Client() + if c == nil { + return nil, nil, errors.New("failed to peek a healthy node to connect to") + } st, err := c.CreateSession(ctx, math.MaxUint64) if err != nil { return nil, nil, errors.Wrap(err, "failed to create reusable neofs session token") From f7007f2085ae54ea976e1ce99cbe0a64b6e9774b Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Mon, 5 Apr 2021 20:13:49 +0300 Subject: [PATCH 121/548] Add connection pool implementation (part 3) Signed-off-by: Pavel Korotkov --- connections/{generator.go => sampler.go} | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) rename connections/{generator.go => sampler.go} (79%) diff --git a/connections/generator.go b/connections/sampler.go similarity index 79% rename from connections/generator.go rename to connections/sampler.go index 435386a..8dced58 100644 --- a/connections/generator.go +++ b/connections/sampler.go @@ -23,15 +23,15 @@ func (wl *workList) pop() int { } func NewSampler(probabilities []float64, source rand.Source) *Sampler { - generator := &Sampler{} + sampler := &Sampler{} var ( small workList large workList ) n := len(probabilities) - generator.randomGenerator = rand.New(source) - generator.probabilities = make([]float64, n) - generator.alias = make([]int, n) + sampler.randomGenerator = rand.New(source) + sampler.probabilities = make([]float64, n) + sampler.alias = make([]int, n) // Compute scaled probabilities. p := make([]float64, n) for i := 0; i < n; i++ { @@ -46,8 +46,8 @@ func NewSampler(probabilities []float64, source rand.Source) *Sampler { } for len(large) > 0 && len(small) > 0 { l, g := small.pop(), large.pop() - generator.probabilities[l] = p[l] - generator.alias[l] = g + sampler.probabilities[l] = p[l] + sampler.alias[l] = g p[g] = (p[g] + p[l]) - 1 if p[g] < 1 { small.push(g) @@ -57,13 +57,13 @@ func NewSampler(probabilities []float64, source rand.Source) *Sampler { } for len(large) > 0 { g := large.pop() - generator.probabilities[g] = 1 + sampler.probabilities[g] = 1 } for len(small) > 0 { l := small.pop() - generator.probabilities[l] = 1 + sampler.probabilities[l] = 1 } - return generator + return sampler } func (g *Sampler) Next() int { From c21324bf775af0c4d1679a018d40786da820c615 Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Tue, 6 Apr 2021 12:06:11 +0300 Subject: [PATCH 122/548] Make minor refactoring Signed-off-by: Pavel Korotkov --- connections/sampler.go | 46 +++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/connections/sampler.go b/connections/sampler.go index 8dced58..e6439af 100644 --- a/connections/sampler.go +++ b/connections/sampler.go @@ -2,26 +2,13 @@ package connections import "math/rand" -// https://www.keithschwarz.com/darts-dice-coins/ +// See Vose's Alias Method (https://www.keithschwarz.com/darts-dice-coins/). type Sampler struct { randomGenerator *rand.Rand probabilities []float64 alias []int } -type workList []int - -func (wl *workList) push(e int) { - *wl = append(*wl, e) -} - -func (wl *workList) pop() int { - l := len(*wl) - 1 - n := (*wl)[l] - *wl = (*wl)[:l] - return n -} - func NewSampler(probabilities []float64, source rand.Source) *Sampler { sampler := &Sampler{} var ( @@ -39,28 +26,28 @@ func NewSampler(probabilities []float64, source rand.Source) *Sampler { } for i, pi := range p { if pi < 1 { - small = append(small, i) + small.add(i) } else { - large = append(large, i) + large.add(i) } } - for len(large) > 0 && len(small) > 0 { - l, g := small.pop(), large.pop() + for len(small) > 0 && len(large) > 0 { + l, g := small.remove(), large.remove() sampler.probabilities[l] = p[l] sampler.alias[l] = g - p[g] = (p[g] + p[l]) - 1 + p[g] = p[g] + p[l] - 1 if p[g] < 1 { - small.push(g) + small.add(g) } else { - large.push(g) + large.add(g) } } for len(large) > 0 { - g := large.pop() + g := large.remove() sampler.probabilities[g] = 1 } for len(small) > 0 { - l := small.pop() + l := small.remove() sampler.probabilities[l] = 1 } return sampler @@ -74,3 +61,16 @@ func (g *Sampler) Next() int { } return g.alias[i] } + +type workList []int + +func (wl *workList) add(e int) { + *wl = append(*wl, e) +} + +func (wl *workList) remove() int { + l := len(*wl) - 1 + n := (*wl)[l] + *wl = (*wl)[:l] + return n +} From d7617110b788ea493a1962a97226c6507b65d636 Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Tue, 6 Apr 2021 12:12:19 +0300 Subject: [PATCH 123/548] Remove test dot-env file Signed-off-by: Pavel Korotkov --- .gitignore | 1 + .test.env | 15 --------------- 2 files changed, 1 insertion(+), 15 deletions(-) delete mode 100644 .test.env diff --git a/.gitignore b/.gitignore index 104aa0a..b3199e5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ cmd/test /plugins/ /vendor/ +.test.env *.log test.sh testfile diff --git a/.test.env b/.test.env deleted file mode 100644 index 6eaea9e..0000000 --- a/.test.env +++ /dev/null @@ -1,15 +0,0 @@ -HTTP_GW_VERBOSE=--true -HTTP_GW_KEY=/home/pk/devbox/work/neo/user.key - -HTTP_GW_LISTEN_ADDRESS=0.0.0.0:8087 -HTTP_GW_LOGGER_LEVEL=debug -HTTP_GW_CONNECT_TIMEOUT=60s -HTTP_GW_REQUEST_TIMEOUT=300s -HTTP_GW_KEEPALIVE_TIMEOUT=300s -HTTP_GW_KEEPALIVE_TIME=120s -HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=True -HTTP_GW_CONN_TTL=1h -HTTP_GW_PEERS_0_WEIGHT=0.4 -HTTP_GW_PEERS_0_ADDRESS=s01.neofs.devenv:8080 -HTTP_GW_PEERS_1_WEIGHT=0.6 -HTTP_GW_PEERS_1_ADDRESS=s02.neofs.devenv:8080 \ No newline at end of file From fad05b76d407bf158ced14e0ba38415a71af2768 Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Wed, 7 Apr 2021 15:54:30 +0300 Subject: [PATCH 124/548] Put artifacts into pool solely Signed-off-by: Pavel Korotkov --- app.go | 2 ++ connections/pool.go | 65 ++++++++++++++++++++++++------------------ downloader/download.go | 53 ++++++++++++++++++---------------- neofs/client-plant.go | 16 ++--------- uploader/upload.go | 12 ++++---- 5 files changed, 78 insertions(+), 70 deletions(-) diff --git a/app.go b/app.go index bbfead4..bcb2537 100644 --- a/app.go +++ b/app.go @@ -2,6 +2,7 @@ package main import ( "context" + "math" "strconv" "github.com/fasthttp/router" @@ -104,6 +105,7 @@ func newApp(ctx context.Context, opt ...Option) App { NodeConnectionTimeout: a.cfg.GetDuration(cfgConTimeout), NodeRequestTimeout: a.cfg.GetDuration(cfgReqTimeout), ClientRebalanceInterval: a.cfg.GetDuration(cfgRebalance), + SessionExpirationEpoch: math.MaxUint64, } pool, err := pb.Build(ctx, opts) if err != nil { diff --git a/connections/pool.go b/connections/pool.go index 48c859b..e705ea1 100644 --- a/connections/pool.go +++ b/connections/pool.go @@ -3,13 +3,14 @@ package connections import ( "context" "crypto/ecdsa" - "errors" "math" "math/rand" "sync" "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/pkg/errors" "google.golang.org/grpc" ) @@ -18,6 +19,7 @@ type PoolBuilderOptions struct { NodeConnectionTimeout time.Duration NodeRequestTimeout time.Duration ClientRebalanceInterval time.Duration + SessionExpirationEpoch uint64 weights []float64 connections []*grpc.ClientConn } @@ -59,47 +61,54 @@ func (pb *PoolBuilder) Build(ctx context.Context, options *PoolBuilderOptions) ( } type Pool interface { - Client() client.Client + ConnectionArtifacts() (client.Client, *token.SessionToken, error) +} + +type clientPack struct { + client client.Client + sessionToken *token.SessionToken + healthy bool } type pool struct { - lock sync.RWMutex - sampler *Sampler - clients []client.Client - healthy []bool + lock sync.RWMutex + sampler *Sampler + clientPacks []*clientPack } func new(ctx context.Context, options *PoolBuilderOptions) (Pool, error) { - n := len(options.weights) - clients := make([]client.Client, n) - healthy := make([]bool, n) + clientPacks := make([]*clientPack, len(options.weights)) for i, con := range options.connections { c, err := client.New(client.WithDefaultPrivateKey(options.Key), client.WithGRPCConnection(con)) if err != nil { return nil, err } - clients[i] = c - healthy[i] = true + st, err := c.CreateSession(ctx, options.SessionExpirationEpoch) + if err != nil { + address := "unknown" + if epi, err := c.EndpointInfo(ctx); err == nil { + address = epi.NodeInfo().Address() + } + return nil, errors.Wrapf(err, "failed to create neofs session token for client %s", address) + } + clientPacks[i] = &clientPack{client: c, sessionToken: st, healthy: true} } source := rand.NewSource(time.Now().UnixNano()) - pool := &pool{ - sampler: NewSampler(options.weights, source), - clients: clients, - healthy: healthy, - } + sampler := NewSampler(options.weights, source) + pool := &pool{sampler: sampler, clientPacks: clientPacks} go func() { ticker := time.NewTimer(options.ClientRebalanceInterval) for range ticker.C { ok := true - for i, client := range pool.clients { + for i, clientPack := range pool.clientPacks { func() { tctx, c := context.WithTimeout(ctx, options.NodeRequestTimeout) defer c() - if _, err := client.EndpointInfo(tctx); err != nil { + if _, err := clientPack.client.EndpointInfo(tctx); err != nil { ok = false } pool.lock.Lock() - pool.healthy[i] = ok + pool.clientPacks[i].healthy = ok pool.lock.Unlock() }() } @@ -109,24 +118,26 @@ func new(ctx context.Context, options *PoolBuilderOptions) (Pool, error) { return pool, nil } -func (p *pool) Client() client.Client { +func (p *pool) ConnectionArtifacts() (client.Client, *token.SessionToken, error) { p.lock.RLock() defer p.lock.RUnlock() - if len(p.clients) == 1 { - if p.healthy[0] { - return p.clients[0] + if len(p.clientPacks) == 1 { + cp := p.clientPacks[0] + if cp.healthy { + return cp.client, cp.sessionToken, nil } - return nil + return nil, nil, errors.New("no healthy client") } var i *int = nil for k := 0; k < 10; k++ { i_ := p.sampler.Next() - if p.healthy[i_] { + if p.clientPacks[i_].healthy { i = &i_ } } if i != nil { - return p.clients[*i] + cp := p.clientPacks[*i] + return cp.client, cp.sessionToken, nil } - return nil + return nil, nil, errors.New("no healthy client") } diff --git a/downloader/download.go b/downloader/download.go index 37da819..8d447b0 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -10,10 +10,8 @@ import ( "sync" "time" - "github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-http-gate/neofs" "github.com/nspcc-dev/neofs-http-gate/tokens" "github.com/pkg/errors" @@ -138,40 +136,35 @@ func (o objectIDs) Slice() []string { } type Downloader struct { - log *zap.Logger - plant neofs.ClientPlant - getOperations struct { - client client.Client - sessionToken *token.SessionToken - } + log *zap.Logger + plant neofs.ClientPlant } func New(ctx context.Context, log *zap.Logger, plant neofs.ClientPlant) (*Downloader, error) { var err error d := &Downloader{log: log, plant: plant} - d.getOperations.client, d.getOperations.sessionToken, err = d.plant.GetReusableArtifacts(ctx) if err != nil { return nil, errors.Wrap(err, "failed to get neofs client's reusable artifacts") } return d, nil } -func (a *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { +func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { return &request{ RequestCtx: ctx, log: log, - objectClient: a.plant.Object(), + objectClient: d.plant.Object(), } } -func (a *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { +func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { var ( err error address = object.NewAddress() cid, _ = c.UserValue("cid").(string) oid, _ = c.UserValue("oid").(string) val = strings.Join([]string{cid, oid}, "/") - log = a.log.With(zap.String("cid", cid), zap.String("oid", oid)) + log = d.log.With(zap.String("cid", cid), zap.String("oid", oid)) ) if err = address.Parse(val); err != nil { log.Error("wrong object address", zap.Error(err)) @@ -180,20 +173,24 @@ func (a *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { } getOpts := getOptionsPool.Get().(*neofs.GetOptions) defer getOptionsPool.Put(getOpts) - getOpts.Client = a.getOperations.client - getOpts.SessionToken = a.getOperations.sessionToken + getOpts.Client, getOpts.SessionToken, err = d.plant.ConnectionArtifacts() + if err != nil { + log.Error("failed to get neofs connection artifacts", zap.Error(err)) + c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) + return + } getOpts.ObjectAddress = address getOpts.Writer = nil - a.newRequest(c, log).receiveFile(getOpts) + d.newRequest(c, log).receiveFile(getOpts) } -func (a *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { +func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { var ( err error scid, _ = c.UserValue("cid").(string) key, _ = c.UserValue("attr_key").(string) val, _ = c.UserValue("attr_val").(string) - log = a.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) + log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ) cid := container.NewID() if err = cid.Parse(scid); err != nil { @@ -203,14 +200,18 @@ func (a *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { } searchOpts := searchOptionsPool.Get().(*neofs.SearchOptions) defer searchOptionsPool.Put(searchOpts) - searchOpts.Client = a.getOperations.client - searchOpts.SessionToken = a.getOperations.sessionToken + searchOpts.Client, searchOpts.SessionToken, err = d.plant.ConnectionArtifacts() + if err != nil { + log.Error("failed to get neofs connection artifacts", zap.Error(err)) + c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) + return + } searchOpts.BearerToken = nil searchOpts.ContainerID = cid searchOpts.Attribute.Key = key searchOpts.Attribute.Value = val var ids []*object.ID - if ids, err = a.plant.Object().Search(c, searchOpts); err != nil { + if ids, err = d.plant.Object().Search(c, searchOpts); err != nil { log.Error("something went wrong", zap.Error(err)) c.Error("something went wrong", fasthttp.StatusBadRequest) return @@ -229,9 +230,13 @@ func (a *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { address.SetObjectID(ids[0]) getOpts := getOptionsPool.Get().(*neofs.GetOptions) defer getOptionsPool.Put(getOpts) - getOpts.Client = a.getOperations.client - getOpts.SessionToken = a.getOperations.sessionToken + getOpts.Client, getOpts.SessionToken, err = d.plant.ConnectionArtifacts() + if err != nil { + log.Error("failed to get neofs connection artifacts", zap.Error(err)) + c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) + return + } getOpts.ObjectAddress = address getOpts.Writer = nil - a.newRequest(c, log).receiveFile(getOpts) + d.newRequest(c, log).receiveFile(getOpts) } diff --git a/neofs/client-plant.go b/neofs/client-plant.go index 1430619..e89fd25 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -5,7 +5,6 @@ import ( "context" "crypto/ecdsa" "io" - "math" "github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/container" @@ -15,7 +14,6 @@ import ( "github.com/nspcc-dev/neofs-http-gate/connections" objectCore "github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/transformer" - "github.com/pkg/errors" ) const maxObjectSize = uint64(1 << 28) // Limit objects to 256 MiB. @@ -62,7 +60,7 @@ type ObjectClient interface { } type ClientPlant interface { - GetReusableArtifacts(ctx context.Context) (client.Client, *token.SessionToken, error) + ConnectionArtifacts() (client.Client, *token.SessionToken, error) Object() ObjectClient OwnerID() *owner.ID } @@ -78,16 +76,8 @@ type neofsClientPlant struct { pool connections.Pool } -func (cp *neofsClientPlant) GetReusableArtifacts(ctx context.Context) (client.Client, *token.SessionToken, error) { - c := cp.pool.Client() - if c == nil { - return nil, nil, errors.New("failed to peek a healthy node to connect to") - } - st, err := c.CreateSession(ctx, math.MaxUint64) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to create reusable neofs session token") - } - return c, st, nil +func (cp *neofsClientPlant) ConnectionArtifacts() (client.Client, *token.SessionToken, error) { + return cp.pool.ConnectionArtifacts() } func (cc *neofsClientPlant) Object() ObjectClient { diff --git a/uploader/upload.go b/uploader/upload.go index 158939c..2b10996 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -56,7 +56,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { return } defer func() { - // if temporary reader can be closed - close it + // If the temporary reader can be closed - let's close it. if file == nil { return } @@ -106,10 +106,10 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { putOpts := putOptionsPool.Get().(*neofs.PutOptions) defer putOptionsPool.Put(putOpts) // Try to put file into NeoFS or throw an error. - putOpts.Client, putOpts.SessionToken, err = u.plant.GetReusableArtifacts(c) + putOpts.Client, putOpts.SessionToken, err = u.plant.ConnectionArtifacts() if err != nil { - log.Error("failed to get neofs client's reusable artifacts", zap.Error(err)) - c.Error("failed to get neofs client's reusable artifacts", fasthttp.StatusInternalServerError) + log.Error("failed to get neofs connection artifacts", zap.Error(err)) + c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) return } putOpts.BearerToken = bt @@ -118,8 +118,8 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { putOpts.PrepareObjectOnsite = false putOpts.Reader = file if addr, err = u.plant.Object().Put(c, putOpts); err != nil { - log.Error("could not store file in NeoFS", zap.Error(err)) - c.Error("could not store file in NeoFS", fasthttp.StatusBadRequest) + log.Error("could not store file in neofs", zap.Error(err)) + c.Error("could not store file in neofs", fasthttp.StatusBadRequest) return } // Try to return the response, otherwise, if something went wrong, throw an error. From e3dbecf8441645ad72e70e7b47d75a5e0c008ea0 Mon Sep 17 00:00:00 2001 From: Pavel Korotkov Date: Thu, 8 Apr 2021 12:37:55 +0300 Subject: [PATCH 125/548] Rework connection artifacts choice logic Signed-off-by: Pavel Korotkov --- connections/pool.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/connections/pool.go b/connections/pool.go index e705ea1..256acfa 100644 --- a/connections/pool.go +++ b/connections/pool.go @@ -128,16 +128,12 @@ func (p *pool) ConnectionArtifacts() (client.Client, *token.SessionToken, error) } return nil, nil, errors.New("no healthy client") } - var i *int = nil - for k := 0; k < 10; k++ { - i_ := p.sampler.Next() - if p.clientPacks[i_].healthy { - i = &i_ + attempts := 3 * len(p.clientPacks) + for k := 0; k < attempts; k++ { + i := p.sampler.Next() + if cp := p.clientPacks[i]; cp.healthy { + return cp.client, cp.sessionToken, nil } } - if i != nil { - cp := p.clientPacks[*i] - return cp.client, cp.sessionToken, nil - } return nil, nil, errors.New("no healthy client") } From b2e7a2cb61efb8c69cf5742bea2f4e974a1d51bc Mon Sep 17 00:00:00 2001 From: Stanislav Bogatyrev Date: Fri, 9 Apr 2021 18:29:18 +0300 Subject: [PATCH 126/548] Clean up Makefile and image build Signed-off-by: Stanislav Bogatyrev --- .gitignore | 4 +- .golangci.yml | 47 +++++++++++++++ Dockerfile | 28 +++------ Dockerfile.dirty | 8 +++ Makefile | 145 +++++++++++++++++++++++++++++++---------------- README.md | 27 ++++----- help.mk | 11 ---- misc.go | 1 + 8 files changed, 173 insertions(+), 98 deletions(-) create mode 100644 .golangci.yml create mode 100644 Dockerfile.dirty delete mode 100644 help.mk diff --git a/.gitignore b/.gitignore index b3199e5..405b1dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ +.idea bin temp -cmd/test /plugins/ /vendor/ @@ -10,3 +10,5 @@ test.sh testfile .blast.yml .neofs-cli.yml + +.cache diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..ba892e7 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,47 @@ +# This file contains all available configuration options +# with their default values. + +# options for analysis running +run: + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 5m + + # include test files or not, default is true + tests: false + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" + format: tab + +# all available settings of specific linters +linters-settings: + exhaustive: + # indicates that switch statements are to be considered exhaustive if a + # 'default' case is present, even if all enum members aren't listed in the + # switch + default-signifies-exhaustive: true + govet: + # report about shadowed variables + check-shadowing: false + +linters: + enable: + # mandatory linters + - govet + - golint + + # some default golangci-lint linters + - errcheck + - gosimple + - ineffassign + - staticcheck + - typecheck + + # extra linters + - exhaustive + - gofmt + - whitespace + - goimports + disable-all: true + fast: false diff --git a/Dockerfile b/Dockerfile index ce850dc..187993e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,23 @@ -FROM golang:1 as builder - +FROM golang:1.16-alpine as basebuilder +RUN apk add --update make bash ca-certificates +FROM basebuilder as builder ENV GOGC off ENV CGO_ENABLED 0 - -RUN set -x \ - && apt update \ - && apt install -y upx-ucl - +ARG BUILD=now +ARG VERSION=dev +ARG REPO=repository WORKDIR /src COPY . /src -ARG VERSION=dev -ENV LDFLAGS "-w -s -X main.Version=${VERSION}" -RUN set -x \ - && go build \ - -v \ - -mod=vendor \ - -trimpath \ - -ldflags "${LDFLAGS} -X main.Build=$(date -u +%s%N)" \ - -o /go/bin/neofs-gw ./ \ - && upx -3 /go/bin/neofs-gw +RUN make # Executable image FROM scratch WORKDIR / -COPY --from=builder /go/bin/neofs-gw /bin/neofs-gw COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /src/bin/neofs-http-gw /bin/neofs-http-gw -ENTRYPOINT ["/bin/neofs-gw"] +ENTRYPOINT ["/bin/neofs-http-gw"] diff --git a/Dockerfile.dirty b/Dockerfile.dirty new file mode 100644 index 0000000..c17734b --- /dev/null +++ b/Dockerfile.dirty @@ -0,0 +1,8 @@ +FROM alpine +RUN apk add --update --no-cache bash ca-certificates + +WORKDIR / + +COPY bin/neofs-http-gw /bin/neofs-http-gw + +CMD ["neofs-http-gw"] diff --git a/Makefile b/Makefile index 0ec592e..77c7436 100644 --- a/Makefile +++ b/Makefile @@ -1,63 +1,108 @@ --include .env --include help.mk +#!/usr/bin/make -f -VERSION ?= "$(shell git describe --tags 2>/dev/null | sed 's/^v//')" +REPO ?= $(shell go list -m) +VERSION ?= $(shell git describe --tags --dirty --always) +BUILD ?= $(shell date -u --iso=seconds) +DEBUG ?= false -GRPC_VERSION=$(shell go list -m google.golang.org/grpc | cut -d " " -f 2) +HUB_IMAGE ?= nspccdev/neofs-http-gw +HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" -HUB_IMAGE=nspccdev/neofs +# List of binaries to build. For now just one. +BINDIR = bin +DIRS = $(BINDIR) +BINS = "$(BINDIR)/neofs-http-gw" -B=\033[0;1m -G=\033[0;92m -R=\033[0m +.PHONY: help all dep clean fmts fmt imports test lint docker/lint -.PHONY: version deps image publish +# Make all binaries +all: $(BINS) -# Show current version -version: - @echo "Current version: $(VERSION)-$(GRPC_VERSION)" +$(BINS): $(DIRS) dep + @echo "⇒ Build $@" + CGO_ENABLED=0 \ + GO111MODULE=on \ + go build -v -trimpath \ + -ldflags "-X main.Version=$(VERSION) \ + -X main.Build=$(BUILD) \ + -X main.Debug=$(DEBUG)" \ + -o $@ ./ -# Check and ensure dependencies -deps: - @printf "${B}${G}⇒ Ensure vendor${R}: " - @go mod tidy -v && echo OK || (echo fail && exit 2) - @printf "${B}${G}⇒ Download requirements${R}: " - @go mod download && echo OK || (echo fail && exit 2) - @printf "${B}${G}⇒ Store vendor localy${R}: " - @go mod vendor && echo OK || (echo fail && exit 2) +$(DIRS): + @echo "⇒ Ensure dir: $@" + @mkdir -p $@ -# Build docker image -image: VERSION?= -image: deps - @echo "${B}${G}⇒ Build GW docker-image with $(GRPC_VERSION) ${R}" +# Pull go dependencies +dep: + @printf "⇒ Download requirements: " + @CGO_ENABLED=0 \ + GO111MODULE=on \ + go mod download && echo OK + @printf "⇒ Tidy requirements: " + @CGO_ENABLED=0 \ + GO111MODULE=on \ + go mod tidy -v && echo OK + +# Run all code formatters +fmts: fmt imports + +# Reformat code +fmt: + @echo "⇒ Processing gofmt check" + @GO111MODULE=on gofmt -s -w ./ + +# Reformat imports +imports: + @echo "⇒ Processing goimports check" + @GO111MODULE=on goimports -w ./ + +# Build clean Docker image +image: + @echo "⇒ Build NeoFS HTTP Gateway docker image " @docker build \ + --build-arg REPO=$(REPO) \ --build-arg VERSION=$(VERSION) \ - -f Dockerfile \ - -t $(HUB_IMAGE)-http-gate:$(VERSION) . + --rm \ + -f Dockerfile \ + -t $(HUB_IMAGE):$(HUB_TAG) . -# Publish docker image -publish: - @echo "${B}${G}⇒ publish docker image ${R}" - @docker push $(HUB_IMAGE)-http-gate:$(VERSION) +# Build dirty Docker image +dirty-image: + @echo "⇒ Build NeoFS HTTP Gateway dirty docker image " + @docker build \ + --build-arg REPO=$(REPO) \ + --build-arg VERSION=$(VERSION) \ + --rm \ + -f Dockerfile.dirty \ + -t $(HUB_IMAGE)-dirty:$(HUB_TAG) . -.PHONY: dev +# Run linters +lint: + @golangci-lint --timeout=5m run -# Build development docker images -dev: VERSIONS?=$(GRPC_VERSION) -dev: - @echo "=> Build multiple images for $(VERSIONS)"; \ - git checkout go.{sum,mod}; \ - for v in $(VERSIONS); do \ - curdir=$$(pwd); \ - echo "=> Checkout gRPC to $${v}"; \ - cd ../grpc-go; \ - git checkout $${v} &> /dev/null || (echo "Release $${v} not found" && exit 2); \ - cd ../neofs-api; \ - git checkout go.{sum,mod}; \ - go get google.golang.org/grpc@$${v}; \ - cd $${curdir}; \ - cp go_dev.mod go.mod; \ - go get google.golang.org/grpc@$${v}; \ - make image VERSION=$(VERSION)-$${v}; \ - git checkout go.{sum,mod}; \ - done \ No newline at end of file +# Run linters in Docker +docker/lint: + docker run --rm -it \ + -v `pwd`:/src \ + -u `stat -c "%u:%g" .` \ + --env HOME=/src \ + golangci/golangci-lint:v1.39 bash -c 'cd /src/ && make lint' + +# Print version +version: + @echo $(VERSION) + +# Show this help prompt +help: + @echo ' Usage:' + @echo '' + @echo ' make ' + @echo '' + @echo ' Targets:' + @echo '' + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u + +# Clean up +clean: + rm -rf vendor + rm -rf $(BINDIR) diff --git a/README.md b/README.md index c28063a..537aeb3 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,18 @@ -# NeoFS HTTP Gate +# NeoFS HTTP Protocol Gateway -NeoFS HTTP Gate is example of tool that provides basic interactions with NeoFS. - -- you can download one file per request from NeoFS Network using NeoFS Gate -- you can upload one file per request into NeoFS Network using NeoFS Gate +NeoFS HTTP Protocol Gateway bridges NeoFS internal protocol and HTTP standard. +- you can download one file per request from NeoFS Network +- you can upload one file per request into the NeoFS Network ## Notable make targets ``` - Usage: - - make - - Targets: - - deps Check and ensure dependencies - dev Build development docker images - help Show this help prompt - image Build docker image - publish Publish docker image - version Show current version +dep Check and ensure dependencies +image Build clean docker image +dirty-image Build diry docker image with host-built binaries +fmts Run all code formatters +lint Run linters +version Show current version ``` ## Install diff --git a/help.mk b/help.mk deleted file mode 100644 index c065ec8..0000000 --- a/help.mk +++ /dev/null @@ -1,11 +0,0 @@ -.PHONY: help - -# Show this help prompt -help: - @echo ' Usage:' - @echo '' - @echo ' make ' - @echo '' - @echo ' Targets:' - @echo '' - @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq diff --git a/misc.go b/misc.go index 1541334..50b8cb0 100644 --- a/misc.go +++ b/misc.go @@ -5,4 +5,5 @@ const Prefix = "HTTP_GW" var ( Build = "now" Version = "dev" + Debug = "false" ) From 11562232316604463445917f8ec8267821342c42 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 13 Apr 2021 12:44:58 +0300 Subject: [PATCH 127/548] go.sum: clean up Make this thing build without go.sum changes. --- go.sum | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/go.sum b/go.sum index 439631f..075d900 100644 --- a/go.sum +++ b/go.sum @@ -42,7 +42,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= -github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= @@ -179,7 +178,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -256,7 +254,6 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo= @@ -303,7 +300,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -331,12 +327,9 @@ github.com/nspcc-dev/dbft v0.0.0-20210302103605-cc75991b7cfb/go.mod h1:U8MSnEShH github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= -github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= github.com/nspcc-dev/neo-go v0.94.0 h1:2eafoyEnueqEMGDZF06HZJQN0MHgYxFzlcre5YUOP1M= github.com/nspcc-dev/neo-go v0.94.0/go.mod h1:IrBT/UG3/Slhqgjge8r6zni5tNRsWmwwG9OnVdASheI= -github.com/nspcc-dev/neofs-api-go v1.23.0 h1:t4FB5uVY99UkYR0Hiyi1SHjZuqzf4qicw7tf7BBnkHk= -github.com/nspcc-dev/neofs-api-go v1.23.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-api-go v1.25.0 h1:hw1TTi3/3wCBB3KN6cycuM8Sn/MFnapSQptBx8KgDIY= github.com/nspcc-dev/neofs-api-go v1.25.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= @@ -344,8 +337,6 @@ github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9K github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-http-gate/sdk v0.3.4 h1:RtYWuF9xDWrkVwu6sFRWlyZ+ToYM7Y9h8B93Fg4CPEA= -github.com/nspcc-dev/neofs-http-gate/sdk v0.3.4/go.mod h1:JC4dT16H5HilyZcb8sTxL/TMC1FSEKMuFAqRsmAPoAk= github.com/nspcc-dev/neofs-node v0.17.0 h1:Xh01n8AKC1XmSRBN7yBKSO7Vz2rchqZfAnofcq1lis0= github.com/nspcc-dev/neofs-node v0.17.0/go.mod h1:nC4+0Zqxv7KZMOv59g5TvcCg0kV6m+1ySiDEuzlAuLc= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= @@ -415,7 +406,6 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= @@ -480,7 +470,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -495,10 +484,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.19.0 h1:PfTS4PeH3xDr3WomrDS2ID8lU2GskK1xS3YG6gIpibU= github.com/valyala/fasthttp v1.19.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= -github.com/valyala/fasthttp v1.20.0 h1:olTmcnLQeZrkBc4TVgE/BatTo1NE/IvW050AuD8SW+U= -github.com/valyala/fasthttp v1.20.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= github.com/valyala/fasthttp v1.22.0 h1:OpwH5KDOJ9cS2bq8fD+KfT4IrksK0llvkHf4MZx42jQ= github.com/valyala/fasthttp v1.22.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= @@ -520,7 +506,6 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -547,9 +532,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -566,8 +548,8 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -594,12 +576,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191105084925-a882066a44e0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226101413-39120d07d75e h1:jIQURUJ9mlLvYwTBtRHm9h58rYhSonLvRvgAnP8Nr7I= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -644,13 +622,10 @@ golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -659,9 +634,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -720,7 +693,6 @@ google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dT google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= @@ -737,10 +709,6 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1 h1:cmUfbeGKnz9+2DD/UYsMQXeqbHZqZDs4eQwW0sFOpBY= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -749,7 +717,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= @@ -773,7 +740,6 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= From c06a3137e5bec47f3bead532bb69b4a742132d79 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 14 Apr 2021 22:57:58 +0300 Subject: [PATCH 128/548] app: add support for ephemeral keys --- README.md | 2 +- app.go | 13 ++++++++++++- neofs/credentials.go | 22 ++++++++++++++++++++++ settings.go | 2 +- 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 537aeb3..6e214cc 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ version Show current version --metrics enable prometheus -h, --help show help -v, --version show version - --key string "generated" to generate key, path to private key file, hex string or wif (default "generated") + --key string path to private key file, hex string or wif (the key will be autogenerated if not specified) --verbose debug gRPC connections --request_timeout duration gRPC request timeout (default 5s) --connect_timeout duration gRPC connect timeout (default 30s) diff --git a/app.go b/app.go index bcb2537..1144de0 100644 --- a/app.go +++ b/app.go @@ -56,6 +56,11 @@ func WithConfig(c *viper.Viper) Option { } func newApp(ctx context.Context, opt ...Option) App { + var ( + creds neofs.Credentials + err error + ) + a := &app{ log: zap.L(), cfg: viper.GetViper(), @@ -86,7 +91,13 @@ func newApp(ctx context.Context, opt ...Option) App { a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - creds, err := neofs.NewCredentials(a.cfg.GetString(cmdNeoFSKey)) + keystring := a.cfg.GetString(cmdNeoFSKey) + if len(keystring) == 0 { + a.log.Info("no key specified, creating one automatically for this run") + creds, err = neofs.NewEphemeralCredentials() + } else { + creds, err = neofs.NewCredentials(keystring) + } if err != nil { a.log.Fatal("failed to get neofs credentials", zap.Error(err)) } diff --git a/neofs/credentials.go b/neofs/credentials.go index 20b8c9b..990aa96 100644 --- a/neofs/credentials.go +++ b/neofs/credentials.go @@ -2,6 +2,9 @@ package neofs import ( "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "math/big" "github.com/nspcc-dev/neofs-api-go/pkg/owner" crypto "github.com/nspcc-dev/neofs-crypto" @@ -33,6 +36,25 @@ func NewCredentials(secret string) (Credentials, error) { return setFromPrivateKey(key) } +// NewEphemeralCredentials creates new private key and Credentials based on that +// key. +func NewEphemeralCredentials() (Credentials, error) { + c := elliptic.P256() + priv, x, y, err := elliptic.GenerateKey(c, rand.Reader) + if err != nil { + return nil, err + } + key := &ecdsa.PrivateKey{ + PublicKey: ecdsa.PublicKey{ + Curve: c, + X: x, + Y: y, + }, + D: new(big.Int).SetBytes(priv), + } + return setFromPrivateKey(key) +} + // PrivateKey returns ecdsa.PrivateKey. func (c *credentials) PrivateKey() *ecdsa.PrivateKey { return c.key diff --git a/settings.go b/settings.go index 7844293..32326cd 100644 --- a/settings.go +++ b/settings.go @@ -104,7 +104,7 @@ func settings() *viper.Viper { help := flags.BoolP(cmdHelp, "h", false, "show help") version := flags.BoolP(cmdVersion, "v", false, "show version") - flags.String(cmdNeoFSKey, "", `path to private key file, hex string or wif`) + flags.String(cmdNeoFSKey, "", `path to private key file, hex string or wif (autogenerated key will be used if not specified)`) flags.Bool(cmdVerbose, false, "debug gRPC connections") flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout") From 0d21ca382fb67220e3e82da25e9c9ba7d7733471 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 14 Apr 2021 22:59:12 +0300 Subject: [PATCH 129/548] neofs: drop WIF() from Credentials It's never used and never needs to be used, the key shouldn't be exported from the gate. --- neofs/credentials.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/neofs/credentials.go b/neofs/credentials.go index 990aa96..abc1ae9 100644 --- a/neofs/credentials.go +++ b/neofs/credentials.go @@ -13,7 +13,6 @@ import ( type ( // Credentials contains methods that needed to work with NeoFS. Credentials interface { - WIF() string Owner() *owner.ID PublicKey() *ecdsa.PublicKey PrivateKey() *ecdsa.PrivateKey @@ -22,7 +21,6 @@ type ( credentials struct { key *ecdsa.PrivateKey ownerID *owner.ID - wif string } ) @@ -70,20 +68,11 @@ func (c *credentials) Owner() *owner.ID { return c.ownerID } -// WIF returns string representation of WIF. -func (c *credentials) WIF() string { - return c.wif -} - func setFromPrivateKey(key *ecdsa.PrivateKey) (*credentials, error) { wallet, err := owner.NEO3WalletFromPublicKey(&key.PublicKey) if err != nil { return nil, err } ownerID := owner.NewIDFromNeo3Wallet(wallet) - wif, err := crypto.WIFEncode(key) - if err != nil { - return nil, err - } - return &credentials{key: key, ownerID: ownerID, wif: wif}, nil + return &credentials{key: key, ownerID: ownerID}, nil } From f019c97ce8ee9700773d87ee2686877ac0a3d1f2 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 14 Apr 2021 23:53:47 +0300 Subject: [PATCH 130/548] connections: normalize weights, make config a bit more human-oriented Current scheme requires people to calculate exact percentages for their nodes. People are bad with pecentage calculations and it's really inconvenient if you have more than 3-4 nodes. What I'd like to have is ability to specify weights like weights, not percentages: .._0_WEIGHT=3 .._1_WEIGHT=1 .._2_WEIGHT=1 and let the gateway calculate things for me. --- README.md | 2 +- connections/pool.go | 5 ++--- settings.go | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6e214cc..8a394c5 100644 --- a/README.md +++ b/README.md @@ -80,5 +80,5 @@ HTTP_GW_WEB_MAX_REQUEST_BODY_SIZE=4194304 - maximum request body size, server r Peers preset: HTTP_GW_PEERS__ADDRESS = string -HTTP_GW_PEERS__WEIGHT = 0..1 (float) +HTTP_GW_PEERS__WEIGHT = float ``` diff --git a/connections/pool.go b/connections/pool.go index 256acfa..f8b8f57 100644 --- a/connections/pool.go +++ b/connections/pool.go @@ -3,7 +3,6 @@ package connections import ( "context" "crypto/ecdsa" - "math" "math/rand" "sync" "time" @@ -40,8 +39,8 @@ func (pb *PoolBuilder) Build(ctx context.Context, options *PoolBuilderOptions) ( for _, w := range pb.weights { totalWeight += w } - if math.Abs(totalWeight-1.0) >= 1e-4 { - return nil, errors.New("total weight must be equal to unity") + for i, w := range pb.weights { + pb.weights[i] = w / totalWeight } var cons = make([]*grpc.ClientConn, len(pb.addresses)) for i, address := range pb.addresses { diff --git a/settings.go b/settings.go index 32326cd..4e4dabb 100644 --- a/settings.go +++ b/settings.go @@ -183,7 +183,7 @@ func settings() *viper.Viper { fmt.Println() fmt.Printf("%s_%s_[N]_ADDRESS = string\n", Prefix, strings.ToUpper(cfgPeers)) - fmt.Printf("%s_%s_[N]_WEIGHT = 0..1 (float)\n", Prefix, strings.ToUpper(cfgPeers)) + fmt.Printf("%s_%s_[N]_WEIGHT = float\n", Prefix, strings.ToUpper(cfgPeers)) os.Exit(0) case version != nil && *version: From a6f63c2bac105e2aae2175288ac1d7634d034351 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 14 Apr 2021 23:57:45 +0300 Subject: [PATCH 131/548] app: use 1 for weight if not specified or wrong If we have a number of nodes with unspecified weights they'd all be treated equal which seems to be fair. --- README.md | 2 +- app.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8a394c5..34c04ad 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ HTTP_GW_REQUEST_TIMEOUT=duration - Timeout for request HTTP_GW_REBALANCE_TIMER=duration - Time between connections checks HTTP_GW_LISTEN_ADDRESS=host:port - Address to listen connections HTTP_GW_PEERS__ADDRESS=host:port - Address of NeoFS Node -HTTP_GW_PEERS__WEIGHT=float - Weight of NeoFS Node +HTTP_GW_PEERS__WEIGHT=float - Weight of NeoFS Node (1 if not specified) HTTP_GW_PPROF=bool - Enable/disable pprof (/debug/pprof) HTTP_GW_METRICS=bool - Enable/disable prometheus metrics endpoint (/metrics) HTTP_GW_LOGGER_FORMAT=string - Logger format diff --git a/app.go b/app.go index 1144de0..bafb90c 100644 --- a/app.go +++ b/app.go @@ -108,6 +108,9 @@ func newApp(ctx context.Context, opt ...Option) App { if address == "" { break } + if weight <= 0 { // unspecified or wrong + weight = 1 + } pb.AddNode(address, weight) a.log.Info("add connection", zap.String("address", address), zap.Float64("weight", weight)) } From 491ae13190121ead10704424ed1256d6362ba43b Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 15 Apr 2021 17:12:24 +0300 Subject: [PATCH 132/548] app: add support for TLS key/cert options Run in TLS mode if anything is specified. --- README.md | 4 ++++ app.go | 13 +++++++++++-- settings.go | 6 +++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 34c04ad..5e5472d 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ version Show current version --request_timeout duration gRPC request timeout (default 5s) --connect_timeout duration gRPC connect timeout (default 30s) --listen_address string HTTP gate's listen address (default "0.0.0.0:8082") + --tls_certificate string TLS certificate path + --tls_key string TLS key path -p, --peers stringArray NeoFS nodes # Environments: @@ -47,6 +49,8 @@ HTTP_GW_CONNECT_TIMEOUT=duration - Timeout for connection HTTP_GW_REQUEST_TIMEOUT=duration - Timeout for request HTTP_GW_REBALANCE_TIMER=duration - Time between connections checks HTTP_GW_LISTEN_ADDRESS=host:port - Address to listen connections +HTTP_GW_TLS_CERTIFICATE=path - File with TLS certificate +HTTP_GW_TLS_KEY=path - File with TLS private key HTTP_GW_PEERS__ADDRESS=host:port - Address of NeoFS Node HTTP_GW_PEERS__WEIGHT=float - Weight of NeoFS Node (1 if not specified) HTTP_GW_PPROF=bool - Enable/disable pprof (/debug/pprof) diff --git a/app.go b/app.go index bafb90c..09d14bc 100644 --- a/app.go +++ b/app.go @@ -178,9 +178,18 @@ func (a *app) Serve(ctx context.Context) { attachProfiler(r) } bind := a.cfg.GetString(cfgListenAddress) - a.log.Info("running web server", zap.String("address", bind)) + tlsCertPath := a.cfg.GetString(cfgTLSCertificate) + tlsKeyPath := a.cfg.GetString(cfgTLSKey) + a.webServer.Handler = r.Handler - if err := a.webServer.ListenAndServe(bind); err != nil { + if tlsCertPath == "" && tlsKeyPath == "" { + a.log.Info("running web server", zap.String("address", bind)) + err = a.webServer.ListenAndServe(bind) + } else { + a.log.Info("running web server (TLS-enabled)", zap.String("address", bind)) + err = a.webServer.ListenAndServeTLS(bind, tlsCertPath, tlsKeyPath) + } + if err != nil { a.log.Fatal("could not start server", zap.Error(err)) } } diff --git a/settings.go b/settings.go index 4e4dabb..ca27a17 100644 --- a/settings.go +++ b/settings.go @@ -26,7 +26,9 @@ const ( defaultKeepaliveTime = 10 * time.Second defaultKeepaliveTimeout = 10 * time.Second - cfgListenAddress = "listen_address" + cfgListenAddress = "listen_address" + cfgTLSCertificate = "tls_certificate" + cfgTLSKey = "tls_key" // KeepAlive cfgKeepaliveTime = "keepalive.time" @@ -112,6 +114,8 @@ func settings() *viper.Viper { flags.Duration(cfgRebalance, defaultRebalanceTimer, "gRPC connection rebalance timer") flags.String(cfgListenAddress, "0.0.0.0:8082", "address to listen") + flags.String(cfgTLSCertificate, "", "TLS certificate path") + flags.String(cfgTLSKey, "", "TLS key path") peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") // set prefers: From ed27e28a306fd1135f66aa10eaf37f8e926da15f Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 15 Apr 2021 17:16:44 +0300 Subject: [PATCH 133/548] app: drop unused Worker/jobDone --- app.go | 14 +------------- main.go | 1 - 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/app.go b/app.go index 09d14bc..d3de2f2 100644 --- a/app.go +++ b/app.go @@ -24,13 +24,11 @@ type ( cfg *viper.Viper auxiliaryLog logger.Logger webServer *fasthttp.Server - jobDone chan struct{} webDone chan struct{} } App interface { Wait() - Worker(context.Context) Serve(context.Context) } @@ -65,7 +63,6 @@ func newApp(ctx context.Context, opt ...Option) App { log: zap.L(), cfg: viper.GetViper(), webServer: new(fasthttp.Server), - jobDone: make(chan struct{}), webDone: make(chan struct{}), } for i := range opt { @@ -134,16 +131,7 @@ func newApp(ctx context.Context, opt ...Option) App { func (a *app) Wait() { a.log.Info("starting application") - select { - case <-a.jobDone: // wait for job is stopped - <-a.webDone - case <-a.webDone: // wait for web-server is stopped - <-a.jobDone - } -} - -func (a *app) Worker(ctx context.Context) { - close(a.jobDone) + <-a.webDone // wait for web-server to be stopped } func (a *app) Serve(ctx context.Context) { diff --git a/main.go b/main.go index 9e07835..66f3f95 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,6 @@ func main() { globalContext := global.Context() app := newApp(globalContext, WithLogger(l), WithConfig(v)) go app.Serve(globalContext) - go app.Worker(globalContext) app.Wait() } From b020cd6652c8e84915ca14d77354aab27aa341c2 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 23 Apr 2021 15:28:58 +0300 Subject: [PATCH 134/548] uploader: fix passing attributes to object --- neofs/client-plant.go | 3 +++ uploader/upload.go | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/neofs/client-plant.go b/neofs/client-plant.go index e89fd25..2f7ed87 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -26,6 +26,7 @@ type BaseOptions struct { type PutOptions struct { BaseOptions + Attributes []*object.Attribute ContainerID *container.ID OwnerID *owner.ID PrepareObjectOnsite bool @@ -105,6 +106,7 @@ func (oc *neofsObjectClient) Put(ctx context.Context, options *PutOptions) (*obj rawObject := objectCore.NewRaw() rawObject.SetContainerID(options.ContainerID) rawObject.SetOwnerID(options.OwnerID) + rawObject.SetAttributes(options.Attributes...) ns := newNetworkState(ctx, options.Client) objectTarget := transformer.NewPayloadSizeLimiter(maxObjectSize, func() transformer.ObjectTarget { return transformer.NewFormatTarget(&transformer.FormatterParams{ @@ -137,6 +139,7 @@ func (oc *neofsObjectClient) Put(ctx context.Context, options *PutOptions) (*obj rawObject := object.NewRaw() rawObject.SetContainerID(options.ContainerID) rawObject.SetOwnerID(options.OwnerID) + rawObject.SetAttributes(options.Attributes...) ops := new(client.PutObjectParams). WithObject(rawObject.Object()). WithPayloadReader(options.Reader) diff --git a/uploader/upload.go b/uploader/upload.go index 2b10996..8cde2c2 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -98,11 +98,6 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { attributes = append(attributes, timestamp) } oid, bt := u.fetchOwnerAndBearerToken(c) - // Prepare a new object and fill it. - raw := object.NewRaw() - raw.SetContainerID(cid) - raw.SetOwnerID(oid) - raw.SetAttributes(attributes...) putOpts := putOptionsPool.Get().(*neofs.PutOptions) defer putOptionsPool.Put(putOpts) // Try to put file into NeoFS or throw an error. @@ -112,6 +107,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) return } + putOpts.Attributes = attributes putOpts.BearerToken = bt putOpts.ContainerID = cid putOpts.OwnerID = oid From 464a0134bd89767080a8b8bf03edbb5c23d74846 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 23 Apr 2021 16:14:17 +0300 Subject: [PATCH 135/548] neofs: drop PrepareObjectOnsite option It's using NeoFS node internal packages which shouldn't ever be used in gate code, it's not adding any value to the code we have (standard api-go interfaces work fine). --- go.mod | 5 +- go.sum | 42 ------------- neofs/client-plant.go | 138 ++++++------------------------------------ uploader/upload.go | 1 - 4 files changed, 24 insertions(+), 162 deletions(-) diff --git a/go.mod b/go.mod index 016df60..cf3c794 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.16 require ( github.com/fasthttp/router v1.3.5 + github.com/mr-tron/base58 v1.1.3 // indirect + github.com/nspcc-dev/neo-go v0.94.0 // indirect github.com/nspcc-dev/neofs-api-go v1.25.0 github.com/nspcc-dev/neofs-crypto v0.3.0 - github.com/nspcc-dev/neofs-node v0.17.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 @@ -17,5 +18,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/zap v1.16.0 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect + golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect + golang.org/x/tools v0.0.0-20200123022218-593de606220b // indirect google.golang.org/grpc v1.36.1 ) diff --git a/go.sum b/go.sum index 075d900..fdb8739 100644 --- a/go.sum +++ b/go.sum @@ -10,7 +10,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -33,8 +32,6 @@ github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzg github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/alecthomas/participle v0.6.0/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY= -github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -99,7 +96,6 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -148,7 +144,6 @@ github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFG github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -198,7 +193,6 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -243,7 +237,6 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -251,11 +244,9 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo= github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -285,8 +276,6 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= -github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -303,12 +292,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr-net v0.1.2/go.mod h1:QsWt3XK/3hwvNxZJp92iMQKME1qHfpYmyIjFVsSOY6Y= -github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= @@ -337,13 +320,9 @@ github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9K github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-node v0.17.0 h1:Xh01n8AKC1XmSRBN7yBKSO7Vz2rchqZfAnofcq1lis0= -github.com/nspcc-dev/neofs-node v0.17.0/go.mod h1:nC4+0Zqxv7KZMOv59g5TvcCg0kV6m+1ySiDEuzlAuLc= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= -github.com/nspcc-dev/tzhash v1.4.0 h1:RVIR+mxOBHl58CE99+DXtE31ylD5PEkZSoWqoj4fVjg= -github.com/nspcc-dev/tzhash v1.4.0/go.mod h1:Z8gp/VZbyWgPhaMp/KTmeoW5UTynp/N60g0jTtSzBws= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -364,10 +343,7 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/panjf2000/ants/v2 v2.3.0/go.mod h1:LtwNaBX6OeF5qRtQlaeGndalVwJlS2ueur7uwoAHbPA= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/paulmach/orb v0.2.1/go.mod h1:91bG5A8qKNOiZtlKc0BqKMB3O5kWfRQorTwo8BZ2B/0= -github.com/paulmach/protoscan v0.2.0/go.mod h1:2c55sl1Hu6/tgRfc8Y8zADsxuSCYC2IrPh0JCqP/yrw= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -389,7 +365,6 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU= github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= @@ -405,7 +380,6 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= @@ -415,7 +389,6 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -449,7 +422,6 @@ github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -457,8 +429,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -478,7 +448,6 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -495,7 +464,6 @@ github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBU go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -505,13 +473,11 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= @@ -527,10 +493,8 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= @@ -570,12 +534,10 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191105084925-a882066a44e0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -618,10 +580,8 @@ golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -646,7 +606,6 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -741,7 +700,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= diff --git a/neofs/client-plant.go b/neofs/client-plant.go index 2f7ed87..7b64d66 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -1,7 +1,6 @@ package neofs import ( - "bytes" "context" "crypto/ecdsa" "io" @@ -12,12 +11,8 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-http-gate/connections" - objectCore "github.com/nspcc-dev/neofs-node/pkg/core/object" - "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/transformer" ) -const maxObjectSize = uint64(1 << 28) // Limit objects to 256 MiB. - type BaseOptions struct { Client client.Client SessionToken *token.SessionToken @@ -26,11 +21,10 @@ type BaseOptions struct { type PutOptions struct { BaseOptions - Attributes []*object.Attribute - ContainerID *container.ID - OwnerID *owner.ID - PrepareObjectOnsite bool - Reader io.Reader + Attributes []*object.Attribute + ContainerID *container.ID + OwnerID *owner.ID + Reader io.Reader } type GetOptions struct { @@ -102,58 +96,23 @@ func (oc *neofsObjectClient) Put(ctx context.Context, options *PutOptions) (*obj objectID *object.ID ) address := object.NewAddress() - if options.PrepareObjectOnsite { - rawObject := objectCore.NewRaw() - rawObject.SetContainerID(options.ContainerID) - rawObject.SetOwnerID(options.OwnerID) - rawObject.SetAttributes(options.Attributes...) - ns := newNetworkState(ctx, options.Client) - objectTarget := transformer.NewPayloadSizeLimiter(maxObjectSize, func() transformer.ObjectTarget { - return transformer.NewFormatTarget(&transformer.FormatterParams{ - Key: oc.key, - NextTarget: &remoteClientTarget{ - ctx: ctx, - client: options.Client, - }, - NetworkState: ns, - }) - }) - if err = ns.LastError(); err != nil { - return nil, err - } - err = objectTarget.WriteHeader(rawObject) - if err != nil { - return nil, err - } - _, err = io.Copy(objectTarget, options.Reader) - if err != nil { - return nil, err - } - var ids *transformer.AccessIdentifiers - ids, err = objectTarget.Close() - if err != nil { - return nil, err - } - address.SetObjectID(ids.SelfID()) - } else { - rawObject := object.NewRaw() - rawObject.SetContainerID(options.ContainerID) - rawObject.SetOwnerID(options.OwnerID) - rawObject.SetAttributes(options.Attributes...) - ops := new(client.PutObjectParams). - WithObject(rawObject.Object()). - WithPayloadReader(options.Reader) - objectID, err = options.Client.PutObject( - ctx, - ops, - client.WithSession(options.SessionToken), - client.WithBearer(options.BearerToken), - ) - if err != nil { - return nil, err - } - address.SetObjectID(objectID) + rawObject := object.NewRaw() + rawObject.SetContainerID(options.ContainerID) + rawObject.SetOwnerID(options.OwnerID) + rawObject.SetAttributes(options.Attributes...) + ops := new(client.PutObjectParams). + WithObject(rawObject.Object()). + WithPayloadReader(options.Reader) + objectID, err = options.Client.PutObject( + ctx, + ops, + client.WithSession(options.SessionToken), + client.WithBearer(options.BearerToken), + ) + if err != nil { + return nil, err } + address.SetObjectID(objectID) address.SetContainerID(options.ContainerID) return address, nil } @@ -200,60 +159,3 @@ func (oc *neofsObjectClient) Delete(ctx context.Context, options *DeleteOptions) ) return err } - -type remoteClientTarget struct { - ctx context.Context - client client.Client - object *object.Object - payload []byte -} - -func (rct *remoteClientTarget) WriteHeader(raw *objectCore.RawObject) error { - rct.object = raw.Object().SDK() - return nil -} - -func (rct *remoteClientTarget) Write(p []byte) (n int, err error) { - rct.payload = append(rct.payload, p...) - return len(p), nil -} - -func (rct *remoteClientTarget) Close() (*transformer.AccessIdentifiers, error) { - id, err := rct.client.PutObject( - rct.ctx, new(client.PutObjectParams). - WithObject(rct.object). - WithPayloadReader(bytes.NewReader(rct.payload)), - ) - if err != nil { - return nil, err - } - return new(transformer.AccessIdentifiers).WithSelfID(id), nil -} - -type networkState struct { - ctx context.Context - client client.Client - lastError error - onError func(error) -} - -func newNetworkState(ctx context.Context, client client.Client) *networkState { - ns := &networkState{ - ctx: ctx, - client: client, - } - ns.onError = func(err error) { ns.lastError = err } - return ns -} - -func (ns *networkState) LastError() error { - return ns.lastError -} - -func (ns *networkState) CurrentEpoch() uint64 { - ce, err := ns.client.NetworkInfo(ns.ctx) - if err != nil { - ns.onError(err) - } - return ce.CurrentEpoch() -} diff --git a/uploader/upload.go b/uploader/upload.go index 8cde2c2..3c0da28 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -111,7 +111,6 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { putOpts.BearerToken = bt putOpts.ContainerID = cid putOpts.OwnerID = oid - putOpts.PrepareObjectOnsite = false putOpts.Reader = file if addr, err = u.plant.Object().Put(c, putOpts); err != nil { log.Error("could not store file in neofs", zap.Error(err)) From b94bfb3b766f6337bb70805d6c33fb69c613803f Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 27 Apr 2021 14:34:02 +0300 Subject: [PATCH 136/548] app: drop FIXME comment We've made long way since a428a0b1b3dd49f4bb5cd7f04d4b1b0017327c2d and it's just no longer relevant, the fix is there in fasthttp version we use and streaming works fine. --- app.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/app.go b/app.go index d3de2f2..5c352e9 100644 --- a/app.go +++ b/app.go @@ -82,9 +82,6 @@ func newApp(ctx context.Context, opt ...Option) App { a.webServer.NoDefaultServerHeader = true a.webServer.NoDefaultContentType = true a.webServer.MaxRequestBodySize = a.cfg.GetInt(cfgWebMaxRequestBodySize) - // -- -- -- -- -- -- FIXME -- -- -- -- -- -- - // Does not work with StreamRequestBody due to bugs with - // readMultipartForm, see https://github.com/valyala/fasthttp/issues/968 a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- From ed3c576b7075e969877e0d011a9ff44dddc555a1 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 28 Apr 2021 11:39:12 +0300 Subject: [PATCH 137/548] uploader: drain body buffer before ending request processing Fixes 2021-04-28T00:03:36.514+0300 debug uploader/upload.go:64 close temporary multipart/form file {"cid": "Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ", "address": "Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m8PtaoricLouCn5zE8hAFr3gZEBDCZFe9BEgVJTSocX", "filename": "vid.mp4"} 2021/04/28 00:03:36 error when serving connection "127.0.0.1:8082"<->"127.0.0.1:41390": error when reading request headers: cannot find http request method in "0\r\n\r\n". Buffer size=5, contents: "0\r\n\r\n" --- uploader/upload.go | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/uploader/upload.go b/uploader/upload.go index 3c0da28..02da473 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -18,7 +18,10 @@ import ( "go.uber.org/zap" ) -const jsonHeader = "application/json; charset=UTF-8" +const ( + jsonHeader = "application/json; charset=UTF-8" + drainBufSize = 4096 +) var putOptionsPool = sync.Pool{ New: func() interface{} { @@ -38,12 +41,14 @@ func New(log *zap.Logger, plant neofs.ClientPlant, enableDefaultTimestamp bool) func (u *Uploader) Upload(c *fasthttp.RequestCtx) { var ( - err error - file MultipartFile - addr *object.Address - cid = container.NewID() - scid, _ = c.UserValue("cid").(string) - log = u.log.With(zap.String("cid", scid)) + err error + file MultipartFile + addr *object.Address + cid = container.NewID() + scid, _ = c.UserValue("cid").(string) + log = u.log.With(zap.String("cid", scid)) + bodyStream = c.RequestBodyStream() + drainBuf = make([]byte, drainBufSize) ) if err = tokens.StoreBearerToken(c); err != nil { log.Error("could not fetch bearer token", zap.Error(err)) @@ -69,7 +74,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { ) }() boundary := string(c.Request.Header.MultipartFormBoundary()) - if file, err = fetchMultipartFile(u.log, c.RequestBodyStream(), boundary); err != nil { + if file, err = fetchMultipartFile(u.log, bodyStream, boundary); err != nil { log.Error("could not receive multipart/form", zap.Error(err)) c.Error("could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return @@ -124,6 +129,18 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { return } + // Multipart is multipart and thus can contain more than one part which + // we ignore at the moment. Also, when dealing with chunked encoding + // the last zero-length chunk might be left unread (because multipart + // reader only cares about its boundary and doesn't look further) and + // it will be (erroneously) interpreted as the start of the next + // pipelined header. Thus we need to drain the body buffer. + for { + _, err = bodyStream.Read(drainBuf) + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } + } // Report status code and content type. c.Response.SetStatusCode(fasthttp.StatusOK) c.Response.Header.SetContentType(jsonHeader) From 88e0963240ab30bb5de97fd35697a1bb62a3a920 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 28 Apr 2021 13:15:22 +0300 Subject: [PATCH 138/548] go.mod: temporarily use nspcc-dev fork of valyala/fasthttp We need proper chunk processing. --- go.mod | 2 ++ go.sum | 9 ++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index cf3c794..d2efa4f 100644 --- a/go.mod +++ b/go.mod @@ -22,3 +22,5 @@ require ( golang.org/x/tools v0.0.0-20200123022218-593de606220b // indirect google.golang.org/grpc v1.36.1 ) + +replace github.com/valyala/fasthttp => github.com/nspcc-dev/fasthttp v1.19.1-0.20210428122823-ab82e78c7994 diff --git a/go.sum b/go.sum index fdb8739..c0beb88 100644 --- a/go.sum +++ b/go.sum @@ -39,7 +39,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= -github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -246,7 +245,6 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo= github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -307,6 +305,8 @@ github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxx github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E= github.com/nspcc-dev/dbft v0.0.0-20210302103605-cc75991b7cfb/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= +github.com/nspcc-dev/fasthttp v1.19.1-0.20210428122823-ab82e78c7994 h1:FBzAFRxTVcvty6csu0ujCJbEChOK9MC0GW9FNvKggqw= +github.com/nspcc-dev/fasthttp v1.19.1-0.20210428122823-ab82e78c7994/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= @@ -453,9 +453,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.19.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= -github.com/valyala/fasthttp v1.22.0 h1:OpwH5KDOJ9cS2bq8fD+KfT4IrksK0llvkHf4MZx42jQ= -github.com/valyala/fasthttp v1.22.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -539,7 +536,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -584,7 +580,6 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= From 4e4f873786bc6676c7de5d4486c50e81de05ddaf Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 29 Apr 2021 11:55:02 +0300 Subject: [PATCH 139/548] misc: gofmt -s --- misc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc.go b/misc.go index 50b8cb0..d6fbe2d 100644 --- a/misc.go +++ b/misc.go @@ -5,5 +5,5 @@ const Prefix = "HTTP_GW" var ( Build = "now" Version = "dev" - Debug = "false" + Debug = "false" ) From d9f9a03625561323c6f705103e4768101dc8f4b1 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 29 Apr 2021 11:55:26 +0300 Subject: [PATCH 140/548] neofs: fix linter warnings neofs/client-plant.go:78:1 golint receiver name cc should be consistent with previous receiver name cp for neofsClientPlant neofs/client-plant.go:85:1 golint receiver name cc should be consistent with previous receiver name cp for neofsClientPlant --- neofs/client-plant.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/neofs/client-plant.go b/neofs/client-plant.go index 7b64d66..e2d6c5c 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -75,15 +75,15 @@ func (cp *neofsClientPlant) ConnectionArtifacts() (client.Client, *token.Session return cp.pool.ConnectionArtifacts() } -func (cc *neofsClientPlant) Object() ObjectClient { +func (cp *neofsClientPlant) Object() ObjectClient { return &neofsObjectClient{ - key: cc.key, - pool: cc.pool, + key: cp.key, + pool: cp.pool, } } -func (cc *neofsClientPlant) OwnerID() *owner.ID { - return cc.ownerID +func (cp *neofsClientPlant) OwnerID() *owner.ID { + return cp.ownerID } func NewClientPlant(ctx context.Context, pool connections.Pool, creds Credentials) (ClientPlant, error) { From 61d61a76cce63e646db65495723add82ec6dd704 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 29 Apr 2021 11:58:36 +0300 Subject: [PATCH 141/548] logger: fix infinite recursive call logger/grpc.go:58:66 staticcheck SA5007: infinite recursive call --- logger/grpc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger/grpc.go b/logger/grpc.go index b163e7c..0b9635e 100644 --- a/logger/grpc.go +++ b/logger/grpc.go @@ -55,6 +55,6 @@ func (z *zapLogger) Fatal(args ...interface{}) { z.log.Fatal(args...) } func (z *zapLogger) Fatalln(args ...interface{}) { z.log.Fatal(args...) } -func (z *zapLogger) Fatalf(format string, args ...interface{}) { z.Fatalf(format, args...) } +func (z *zapLogger) Fatalf(format string, args ...interface{}) { z.log.Fatalf(format, args...) } func (z *zapLogger) V(int) bool { return z.Enabled(zapcore.DebugLevel) } From ba293a3ff9f62ed4ed504422f1c7d03231d5f8b8 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 29 Apr 2021 17:02:15 +0300 Subject: [PATCH 142/548] connections: refuse to build zero-length pool Prevent starting a gateway with no nodes configured. --- connections/pool.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/connections/pool.go b/connections/pool.go index f8b8f57..8e1db20 100644 --- a/connections/pool.go +++ b/connections/pool.go @@ -35,6 +35,9 @@ func (pb *PoolBuilder) AddNode(address string, weight float64) *PoolBuilder { } func (pb *PoolBuilder) Build(ctx context.Context, options *PoolBuilderOptions) (Pool, error) { + if len(pb.addresses) == 0 { + return nil, errors.New("no NeoFS peers configured") + } totalWeight := 0.0 for _, w := range pb.weights { totalWeight += w From afbb9d51f16413bc2e12aff6e8cb5133329f5c22 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 29 Apr 2021 18:32:01 +0300 Subject: [PATCH 143/548] *: drop github.com/pkg/errors dependency Use standard error wrapping/unwrapping instead. The conversion is mostly straightforward, but see grpc/grpc-go#2934 for GRPC `status.FromError`, it doesn't currently support unwrapping/errors.As(), so we're unwrapping manually here. --- connections/pool.go | 5 +++-- downloader/download.go | 15 ++++++++++----- go.mod | 1 - tokens/bearer-token.go | 7 ++++--- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/connections/pool.go b/connections/pool.go index 8e1db20..15e22f4 100644 --- a/connections/pool.go +++ b/connections/pool.go @@ -3,13 +3,14 @@ package connections import ( "context" "crypto/ecdsa" + "errors" + "fmt" "math/rand" "sync" "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/token" - "github.com/pkg/errors" "google.golang.org/grpc" ) @@ -91,7 +92,7 @@ func new(ctx context.Context, options *PoolBuilderOptions) (Pool, error) { if epi, err := c.EndpointInfo(ctx); err == nil { address = epi.NodeInfo().Address() } - return nil, errors.Wrapf(err, "failed to create neofs session token for client %s", address) + return nil, fmt.Errorf("failed to create neofs session token for client %s: %w", address, err) } clientPacks[i] = &clientPack{client: c, sessionToken: st, healthy: true} } diff --git a/downloader/download.go b/downloader/download.go index 8d447b0..f52861b 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -2,6 +2,8 @@ package downloader import ( "context" + "errors" + "fmt" "io" "net/http" "path" @@ -14,7 +16,6 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-http-gate/neofs" "github.com/nspcc-dev/neofs-http-gate/tokens" - "github.com/pkg/errors" "github.com/valyala/fasthttp" "go.uber.org/zap" "google.golang.org/grpc/codes" @@ -84,10 +85,14 @@ func (r *request) receiveFile(options *neofs.GetOptions) { zap.Error(err), ) var ( - msg = errors.Wrap(err, "could not receive object").Error() - code = fasthttp.StatusBadRequest + msg = fmt.Sprintf("could not receive object: %v", err) + code = fasthttp.StatusBadRequest + cause = err ) - if st, ok := status.FromError(errors.Cause(err)); ok && st != nil { + for unwrap := errors.Unwrap(err); unwrap != nil; unwrap = errors.Unwrap(cause) { + cause = unwrap + } + if st, ok := status.FromError(cause); ok && st != nil { if st.Code() == codes.NotFound { code = fasthttp.StatusNotFound } @@ -144,7 +149,7 @@ func New(ctx context.Context, log *zap.Logger, plant neofs.ClientPlant) (*Downlo var err error d := &Downloader{log: log, plant: plant} if err != nil { - return nil, errors.Wrap(err, "failed to get neofs client's reusable artifacts") + return nil, fmt.Errorf("failed to get neofs client's reusable artifacts: %w", err) } return d, nil } diff --git a/go.mod b/go.mod index d2efa4f..8c2a78b 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/nspcc-dev/neo-go v0.94.0 // indirect github.com/nspcc-dev/neofs-api-go v1.25.0 github.com/nspcc-dev/neofs-crypto v0.3.0 - github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index 337c604..86e21f0 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -4,9 +4,10 @@ import ( "bytes" "context" "encoding/base64" + "errors" + "fmt" "github.com/nspcc-dev/neofs-api-go/pkg/token" - "github.com/pkg/errors" "github.com/valyala/fasthttp" ) @@ -77,10 +78,10 @@ func fetchBearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { if buf = parse(&ctx.Request.Header); buf == nil { continue } else if data, err := base64.StdEncoding.DecodeString(string(buf)); err != nil { - lastErr = errors.Wrap(err, "could not fetch marshaled from base64") + lastErr = fmt.Errorf("can't base64-decode bearer token: %w", err) continue } else if err = tkn.Unmarshal(data); err != nil { - lastErr = errors.Wrap(err, "could not unmarshal bearer token") + lastErr = fmt.Errorf("can't unmarshal bearer token: %w", err) continue } else if tkn == nil { continue From f99f9e88a7e671aa066580cbcafdd649250eaef9 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 29 Apr 2021 22:58:54 +0300 Subject: [PATCH 144/548] connections: make use of keepalive options --- app.go | 3 +++ connections/pool.go | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/app.go b/app.go index 5c352e9..b42343b 100644 --- a/app.go +++ b/app.go @@ -114,6 +114,9 @@ func newApp(ctx context.Context, opt ...Option) App { NodeRequestTimeout: a.cfg.GetDuration(cfgReqTimeout), ClientRebalanceInterval: a.cfg.GetDuration(cfgRebalance), SessionExpirationEpoch: math.MaxUint64, + KeepaliveTime: a.cfg.GetDuration(cfgKeepaliveTime), + KeepaliveTimeout: a.cfg.GetDuration(cfgKeepaliveTimeout), + KeepalivePermitWoStream: a.cfg.GetBool(cfgKeepalivePermitWithoutStream), } pool, err := pb.Build(ctx, opts) if err != nil { diff --git a/connections/pool.go b/connections/pool.go index 15e22f4..5765d5f 100644 --- a/connections/pool.go +++ b/connections/pool.go @@ -12,6 +12,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/token" "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" ) type PoolBuilderOptions struct { @@ -19,6 +20,9 @@ type PoolBuilderOptions struct { NodeConnectionTimeout time.Duration NodeRequestTimeout time.Duration ClientRebalanceInterval time.Duration + KeepaliveTime time.Duration + KeepaliveTimeout time.Duration + KeepalivePermitWoStream bool SessionExpirationEpoch uint64 weights []float64 connections []*grpc.ClientConn @@ -51,7 +55,15 @@ func (pb *PoolBuilder) Build(ctx context.Context, options *PoolBuilderOptions) ( con, err := func() (*grpc.ClientConn, error) { toctx, c := context.WithTimeout(ctx, options.NodeConnectionTimeout) defer c() - return grpc.DialContext(toctx, address, grpc.WithInsecure(), grpc.WithBlock()) + return grpc.DialContext(toctx, address, + grpc.WithInsecure(), + grpc.WithBlock(), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: options.KeepaliveTime, + Timeout: options.KeepaliveTimeout, + PermitWithoutStream: options.KeepalivePermitWoStream, + }), + ) }() if err != nil { return nil, err From 91c894cb8ce306efa4e2d51a9d819aa35b2e36b0 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 29 Apr 2021 23:09:37 +0300 Subject: [PATCH 145/548] downloader: set object/container/owner x-attributes after other ones Avoid name clashes with regular attributes. --- downloader/download.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index f52861b..60fca76 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -105,9 +105,6 @@ func (r *request) receiveFile(options *neofs.GetOptions) { dis = "attachment" } r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) - r.Response.Header.Set("x-object-id", obj.ID().String()) - r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) - r.Response.Header.Set("x-container-id", obj.ContainerID().String()) for _, attr := range obj.Attributes() { key := attr.Key() val := attr.Value() @@ -128,6 +125,9 @@ func (r *request) receiveFile(options *neofs.GetOptions) { time.Unix(value, 0).Format(time.RFC1123)) } } + r.Response.Header.Set("x-object-id", obj.ID().String()) + r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) + r.Response.Header.Set("x-container-id", obj.ContainerID().String()) r.SetContentType(writer.contentType) r.Response.Header.Set("Content-Disposition", dis+"; filename="+path.Base(filename)) } From 7a4691778165cceb60a2b217f6db5d322aa21745 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 29 Apr 2021 23:46:38 +0300 Subject: [PATCH 146/548] downloader: limit headers sent to the client to some valid subset fasthttp doesn't do complete filtering/escaping for us, thus filter here. --- downloader/download.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/downloader/download.go b/downloader/download.go index 60fca76..037710d 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -63,6 +63,28 @@ func (d *detector) Write(data []byte) (int, error) { return d.Writer.Write(data) } +func isValidToken(s string) bool { + for _, c := range s { + if c <= ' ' || c > 127 { + return false + } + if strings.ContainsRune("()<>@,;:\\\"/[]?={}", c) { + return false + } + } + return true +} + +func isValidValue(s string) bool { + for _, c := range s { + // HTTP specification allows for more technically, but we don't want to escape things. + if c < ' ' || c > 127 || c == '"' { + return false + } + } + return true +} + func (r *request) receiveFile(options *neofs.GetOptions) { var ( err error @@ -108,6 +130,9 @@ func (r *request) receiveFile(options *neofs.GetOptions) { for _, attr := range obj.Attributes() { key := attr.Key() val := attr.Value() + if !isValidToken(key) || !isValidValue(val) { + continue + } r.Response.Header.Set("x-"+key, val) switch key { case object.AttributeFileName: From e80d147d722d149e392e332af2ff4fee4278b1d9 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 00:15:04 +0300 Subject: [PATCH 147/548] README: rewrite all documentation I think we can fit into one page with this, thus how-to-check.md was removed (and it's a bit bloated to me anyway). --- README.md | 295 +++++++++++++++++++++++++++++++++++-------- docs/how-to-check.md | 111 ---------------- 2 files changed, 239 insertions(+), 167 deletions(-) delete mode 100644 docs/how-to-check.md diff --git a/README.md b/README.md index 5e5472d..d15a01f 100644 --- a/README.md +++ b/README.md @@ -4,57 +4,139 @@ NeoFS HTTP Protocol Gateway bridges NeoFS internal protocol and HTTP standard. - you can download one file per request from NeoFS Network - you can upload one file per request into the NeoFS Network -## Notable make targets +## Installation + +```go get -u github.com/nspcc-dev/neofs-http-gate``` + +Or you can call `make` to build it from the cloned repository (the binary will +end up in `bin/neofs-http-gw`). + +### Notable make targets ``` dep Check and ensure dependencies image Build clean docker image -dirty-image Build diry docker image with host-built binaries +dirty-image Build dirty docker image with host-built binaries fmts Run all code formatters lint Run linters version Show current version ``` -## Install +## Execution -```go get -u github.com/nspcc-dev/neofs-http-gate``` +HTTP gateway itself is not a NeoFS node, so to access NeoFS it uses node's +gRPC interface and you need to provide some node that it will connect to. This +can be done either via `-p` parameter or via `HTTP_GW_PEERS__ADDRESS` and +`HTTP_GW_PEERS__WEIGHT` environment variables (the gate supports multiple +NeoFS nodes with weighted load balancing). -## File uploading behaviors +These two commands are functionally equivalent, they run the gate with one +backend node (and otherwise default settings): +``` +$ neofs-http-gw -p 192.168.130.72:8080 +$ HTTP_GW_PEERS_0_ADDRESS=192.168.130.72:8080 neofs-http-gw +``` -- you can upload on file per request -- if `FileName` not provided by Header attributes, multipart/form filename will be used instead +### Configuration -## Configuration +In general, everything available as CLI parameter can also be specified via +environment variables, so they're not specifically mentioned in most cases +(see `--help` also). + +#### Nodes and weights + +You can specify multiple `-p` options to add more NeoFS nodes, this will make +gateway spread requests equally among them (using weight 1 for every node): ``` -# Flags: +$ neofs-http-gw -p 192.168.130.72:8080 -p 192.168.130.71:8080 +``` +If you want some specific load distribution proportions, use weights, but they +can only be specified via environment variables: - --pprof enable pprof - --metrics enable prometheus - -h, --help show help - -v, --version show version - --key string path to private key file, hex string or wif (the key will be autogenerated if not specified) - --verbose debug gRPC connections - --request_timeout duration gRPC request timeout (default 5s) - --connect_timeout duration gRPC connect timeout (default 30s) - --listen_address string HTTP gate's listen address (default "0.0.0.0:8082") - --tls_certificate string TLS certificate path - --tls_key string TLS key path - -p, --peers stringArray NeoFS nodes +``` +$ HTTP_GW_PEERS_0_ADDRESS=192.168.130.72:8080 HTTP_GW_PEERS_0_WEIGHT=9 \ + HTTP_GW_PEERS_1_ADDRESS=192.168.130.71:8080 HTTP_GW_PEERS_1_WEIGHT=1 neofs-http-gw +``` +This command will make gateway use 192.168.130.72 for 90% of requests and +192.168.130.71 for remaining 10%. -# Environments: +#### Keys -HTTP_GW_KEY=string - Path to private key file, hex string or wif string -HTTP_GW_CONNECT_TIMEOUT=duration - Timeout for connection -HTTP_GW_REQUEST_TIMEOUT=duration - Timeout for request -HTTP_GW_REBALANCE_TIMER=duration - Time between connections checks -HTTP_GW_LISTEN_ADDRESS=host:port - Address to listen connections -HTTP_GW_TLS_CERTIFICATE=path - File with TLS certificate -HTTP_GW_TLS_KEY=path - File with TLS private key -HTTP_GW_PEERS__ADDRESS=host:port - Address of NeoFS Node -HTTP_GW_PEERS__WEIGHT=float - Weight of NeoFS Node (1 if not specified) -HTTP_GW_PPROF=bool - Enable/disable pprof (/debug/pprof) -HTTP_GW_METRICS=bool - Enable/disable prometheus metrics endpoint (/metrics) +By default gateway autogenerates key pair it will use for NeoFS requests. If +for some reason you need to have static keys you can pass them via `--key` +parameter. The key can be a path to private key file (as raw bytes), a hex +string or (unencrypted) WIF string. Example: + +``` +$ neofs-http-gw -p 192.168.130.72:8080 -k KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr +``` + +#### Binding and TLS + +Gateway binds to `0.0.0.0:8082` by default and you can change that with +`--listen_address` option. + +It can also provide TLS interface for its users, just specify paths to key and +certificate files via `--tls_key` and `--tls_certificate` parameters. Note +that using these options makes gateway TLS-only, if you need to serve both TLS +and plain text HTTP you either have to run two gateway instances or use some +external redirecting solution. + +Example to bind to `192.168.130.130:443` and serve TLS there: + +``` +$ neofs-http-gw -p 192.168.130.72:8080 --listen_address 192.168.130.130:443 \ + --tls_key=key.pem --tls_certificate=cert.pem +``` + +#### HTTP parameters + +You can tune HTTP read and write buffer sizes as well as timeouts with +`HTTP_GW_WEB_READ_BUFFER_SIZE`, `HTTP_GW_WEB_READ_TIMEOUT`, +`HTTP_GW_WEB_WRITE_BUFFER_SIZE` and `HTTP_GW_WEB_WRITE_TIMEOUT` environment +variables. + +`HTTP_GW_WEB_STREAM_REQUEST_BODY` environment variable can be used to disable +request body streaming (effectively it'll make gateway accept file completely +first and only then try sending it to NeoFS). + +`HTTP_GW_WEB_MAX_REQUEST_BODY_SIZE` controls maximum request body size +limiting uploads to files slightly lower than this limit. + +#### NeoFS parameters + +Gateway can automatically set timestamps for uploaded files based on local +time source, use `HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP` environment +variable to control this behavior. + +#### Monitoring and metrics + +Pprof and Prometheus are integrated into the gateway, but not enabled by +default. To enable them use `--pprof` and `--metrics` flags or +`HTTP_GW_PPROF`/`HTTP_GW_METRICS` environment variables. + +#### Timeouts + +You can tune gRPC interface parameters with `--connect_timeout` (for +connection to node) and `--request_timeout` (for request processing over +established connection) options as well as `HTTP_GW_KEEPALIVE_TIME` +(peer pinging interval), `HTTP_GW_KEEPALIVE_TIMEOUT` (peer pinging timeout) +and `HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM` environment variables. + +gRPC-level checks allow gateway to detect dead peers, but it declares them +unhealthy at pool level once per `--rebalance_timer` interval, so check for it +if needed. + +All timing options accept values with suffixes, so "15s" is 15 seconds and +"2m" is 2 minutes. + +#### Logging + +`--verbose` flag enables gRPC logging and there is a number of environment +variables to tune logging behavior: + +``` HTTP_GW_LOGGER_FORMAT=string - Logger format HTTP_GW_LOGGER_LEVEL=string - Logger level HTTP_GW_LOGGER_NO_CALLER=bool - Logger don't show caller @@ -62,27 +144,128 @@ HTTP_GW_LOGGER_NO_DISCLAIMER=bool - Logger don't show application HTTP_GW_LOGGER_SAMPLING_INITIAL=int - Logger sampling initial HTTP_GW_LOGGER_SAMPLING_THEREAFTER=int - Logger sampling thereafter HTTP_GW_LOGGER_TRACE_LEVEL=string - Logger show trace on level -HTTP_GW_KEEPALIVE_TIME=duration - After a duration of this time if the client sees no activity - it pings the server to see if the transport is still alive -HTTP_GW_KEEPALIVE_TIMEOUT=duration - After having pinged for keepalive check, the client waits for a duration - of Timeout and if no activity is seen even after that the connection - is closed -HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM=bool - If true, client sends keepalive pings even with no active RPCs. - If false, when there are no active RPCs, Time and Timeout will be - ignored and no keepalive pings will be sent -HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=bool - Enable/disable adding current timestamp attribute when object uploads - -HTTP_GW_WEB_READ_BUFFER_SIZE=4096 - per-connection buffer size for requests' reading -HTTP_GW_WEB_READ_TIMEOUT=15s - an amount of time allowed to read the full request including body -HTTP_GW_WEB_WRITE_BUFFER_SIZE=4096 - per-connection buffer size for responses' writing -HTTP_GW_WEB_WRITE_TIMEOUT=1m0s - maximum duration before timing out writes of the response -HTTP_GW_WEB_STREAM_REQUEST_BODY=true - enables request body streaming, and calls the handler sooner when given - body is larger then the current limit -HTTP_GW_WEB_MAX_REQUEST_BODY_SIZE=4194304 - maximum request body size, server rejects requests with bodies exceeding - this limit - -Peers preset: - -HTTP_GW_PEERS__ADDRESS = string -HTTP_GW_PEERS__WEIGHT = float ``` + +## HTTP API provided + +This gateway intentionally provides limited feature set and doesn't try to +substitute (or completely wrap) regular gRPC NeoFS interface. You can download +and upload objects with it, but deleting, searching, managing ACLs, creating +containers and other activities are not supported and not planned to be +supported. + +### Downloading + +#### Requests + +Basic downloading involves container and object ID and is done via GET +requests to `/get/$CID/$OID` path, like this: + +``` +$ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m8PtaoricLouCn5zE8hAFr3gZEBDCZFe9BEgVJTSocY + +``` + +There is also more complex interface provided for attribute-based downloads, +it's usually used to retrieve files by their names, but any other attribute +can be used as well. The generic syntax for it looks like this: + +```/get_by_attribute/$CID/$ATTRIBUTE_NAME/$ATTRIBUTE_VALUE``` + +where `$CID` is a container ID, `$ATTRIBUTE_NAME` is the name of the attribute +we want to use and `ATTRIBUTE_VALUE` is the value of this attribute that the +target object should have. + +If multiple objects have specified attribute with specified value, then the +first one of them is returned (and you can't get others via this interface). + +Example for file name attribute: + +``` +$ wget http://localhost:8082/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/FileName/cat.jpeg +``` + +Some other user-defined attribute: + +``` +$ wget http://localhost:8082/get_by_attribute/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/Ololo/100500 +``` + +An optional `download=true` argument for `Content-Disposition` management is +also supported (more on that below): + +``` +$ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m8PtaoricLouCn5zE8hAFr3gZEBDCZFe9BEgVJTSocY?download=true + +``` + +#### Replies + +You get object contents in the reply body, but at the same time you also get a +set of reply headers generated using the following rules: + * `Content-Length` is set to the length of the object + * `Content-Type` is autodetected dynamically by gateway + * `Content-Disposition` is `inline` for regular requests and `attachment` for + requests with `download=true` argument, `filename` is also added if there + is `FileName` attribute set for this object + * `Last-Modified` header is set to `Timestamp` attribute value if it's + present for the object + * `x-container-id` contains container ID + * `x-object-id` contains object ID + * `x-owner-id` contains owner address + * all the other NeoFS attributes are converted to `x-*` attributes (but only + if they can be safely represented in HTTP header), for example `FileName` + attribute becomes `x-FileName` + +### Uploading + +You can POST files to `/upload/$CID` path where `$CID` is container ID. The +request must contain multipart form with mandatory `filename` parameter. Only +one part in multipart form will be processed, so to upload another file just +issue new POST request. + +Example request: + +``` +$ curl -F 'file=@cat.jpeg;filename=cat.jpeg' http://localhost:8082/upload/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ +``` + +Chunked encoding is supported by the server (but check for request read +timeouts if you're planning some streaming). You can try streaming support +with large file piped through named FIFO pipe: + +``` +$ mkfifo pipe +$ cat video.mp4 > pipe & +$ curl --no-buffer -F 'file=@pipe;filename=catvideo.mp4' http://localhost:8082/upload/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ +``` + +You can also add some attributes to your file using the following rules: + * all "X-Attribute-*" headers get converted to object attributes with + "X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo: + 100500" header to your request the resulting object will get "Ololo: + 100500" attribute + * "X-Attribute-NEOFS-*" headers are special, they're used to set internal + NeoFS attributes starting with `__NEOFS__` prefix, for these attributes all + dashes get converted to underscores and all letters are capitalized. For + example, you can use "X-Attribute-NEOFS-Expiration-Epoch" header to set + `__NEOFS__EXPIRATION_EPOCH` attribute + * `FileName` attribute is set from multipart's `filename` if not set + explicitly via `X-Attribute-FileName` header + * `Timestamp` attribute can be set using gateway local time if using + HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP option and if request doesn't + provide `X-Attribute-Timestamp` header of its own + +For successful uploads you get JSON data in reply body with container and +object ID, like this: +``` +{ + "object_id": "9ANhbry2ryjJY1NZbcjryJMRXG5uGNKd73kD3V1sVFsX", + "container_id": "Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ" +} +``` + +### Metrics and Pprof + +If enabled, Prometheus metrics are available at `/metrics/` path and Pprof at +`/debug/pprof`. diff --git a/docs/how-to-check.md b/docs/how-to-check.md deleted file mode 100644 index ca96116..0000000 --- a/docs/how-to-check.md +++ /dev/null @@ -1,111 +0,0 @@ -# How to check - -1. Create container -``` -→ neofs-cli -k /path/to/user.key -r s01.neofs.devenv:8080 container create --name TestStorage --basic-acl public -p "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" --await -container ID: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM -awaiting... -container has been persisted on sidechain -``` - -2. Put object into container - -``` -→ neofs-cli -k /path/to/user.key -r s01.neofs.devenv:8080 object put --cid 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM --file /path/to/1.jpeg -[/path/to/1.jpeg] Object successfully stored - ID: GTUokhLMtEq1Kh1nzSVCsWybFvVHFQyhZHaVXZBrYd3W - CID: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM -``` - -3. Check that object can be fetched by oldest API - -``` -→ curl -sSI -XGET http://http.neofs.devenv/get/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/GTUokhLMtEq1Kh1nzSVCsWybFvVHFQyhZHaVXZBrYd3W -HTTP/1.1 200 OK -Date: Thu, 03 Dec 2020 15:04:52 GMT -Content-Type: image/jpeg -Content-Length: 93077 -x-object-id: GTUokhLMtEq1Kh1nzSVCsWybFvVHFQyhZHaVXZBrYd3W -x-owner-id: NTrezR3C4X8aMLVg7vozt5wguyNfFhwuFx -x-container-id: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM -x-FileName: 1.jpeg -x-Timestamp: 1607006318 -Last-Modified: Thu, 03 Dec 2020 17:38:38 MSK -Content-Disposition: inline; filename=1.jpeg -``` - -4. Check that object can be fetched by newest API - -``` -→ curl -sSI -XGET http://http.neofs.devenv/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/FileName/1.jpeg -HTTP/1.1 200 OK -Date: Thu, 03 Dec 2020 15:04:52 GMT -Content-Type: image/jpeg -Content-Length: 93077 -x-object-id: GTUokhLMtEq1Kh1nzSVCsWybFvVHFQyhZHaVXZBrYd3W -x-owner-id: NTrezR3C4X8aMLVg7vozt5wguyNfFhwuFx -x-container-id: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM -x-FileName: 1.jpeg -x-Timestamp: 1607006318 -Last-Modified: Thu, 03 Dec 2020 17:38:38 MSK -Content-Disposition: inline; filename=1.jpeg -``` - -5. Put second object with same name - -``` -→ neofs-cli -k /path/to/user.key -r s01.neofs.devenv:8080 object put --cid 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM --file /path/to/1.jpeg -[/path/to/1.jpeg] Object successfully stored - ID: 14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga - CID: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM - -``` - -6. Check that object can be fetched by oldest API - -``` -→ curl -sSI -XGET http://http.neofs.devenv/get/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga -HTTP/1.1 200 OK -Date: Thu, 03 Dec 2020 15:07:51 GMT -Content-Type: image/jpeg -Content-Length: 93077 -x-object-id: 14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga -x-owner-id: NTrezR3C4X8aMLVg7vozt5wguyNfFhwuFx -x-container-id: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM -x-FileName: 1.jpeg -x-Timestamp: 1607006355 -Last-Modified: Thu, 03 Dec 2020 17:39:15 MSK -Content-Disposition: inline; filename=1.jpeg -``` - -7. Retry fetch object by newest API - -``` -→ curl -sSI -XGET http://http.neofs.devenv/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/FileName/1.jpeg -HTTP/1.1 200 OK -Date: Thu, 03 Dec 2020 15:04:28 GMT -Content-Type: image/jpeg -Content-Length: 93077 -x-object-id: 14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga -x-owner-id: NTrezR3C4X8aMLVg7vozt5wguyNfFhwuFx -x-container-id: 88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM -x-FileName: 1.jpeg -x-Timestamp: 1607006355 -Last-Modified: Thu, 03 Dec 2020 17:39:15 MSK -Content-Disposition: inline; filename=1.jpeg -``` - -**http-gate log when find multiple objects** -``` -2020-12-03T18:04:28.617+0300 debug neofs-gw/receive.go:191 find multiple objects {"cid": "88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM", "attr_key": "FileName", "attr_val": "1.jpeg", "object_ids": ["14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga", "GTUokhLMtEq1Kh1nzSVCsWybFvVHFQyhZHaVXZBrYd3W"], "show_object_id": "14Q3AhJhPyJzWrmiYMzswRDY4cXSUgKPSAEDxadkHKga"} -``` - -8. Check newest API when object not found - -``` -→ curl -sSI -XGET http://http.neofs.devenv/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/FileName/2.jpeg -HTTP/1.1 404 Not Found -Date: Thu, 03 Dec 2020 15:11:07 GMT -Content-Type: text/plain; charset=utf-8 -Content-Length: 9 -``` From 293debe6db94fcdfbd96d96b975f50e1df50f6df Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 12:55:51 +0300 Subject: [PATCH 148/548] downloader: ensure downloader/uploader header symmetry Use X-Attribute-* everywhere. --- README.md | 4 ++-- downloader/download.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d15a01f..79a387a 100644 --- a/README.md +++ b/README.md @@ -213,9 +213,9 @@ set of reply headers generated using the following rules: * `x-container-id` contains container ID * `x-object-id` contains object ID * `x-owner-id` contains owner address - * all the other NeoFS attributes are converted to `x-*` attributes (but only + * all the other NeoFS attributes are converted to `X-Attribute-*` headers (but only if they can be safely represented in HTTP header), for example `FileName` - attribute becomes `x-FileName` + attribute becomes `X-Attribute-FileName` header ### Uploading diff --git a/downloader/download.go b/downloader/download.go index 037710d..05a3bde 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -133,7 +133,7 @@ func (r *request) receiveFile(options *neofs.GetOptions) { if !isValidToken(key) || !isValidValue(val) { continue } - r.Response.Header.Set("x-"+key, val) + r.Response.Header.Set("X-Attribute-"+key, val) switch key { case object.AttributeFileName: filename = val From 90b0613182f90d49931626323974804471ffbbb8 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 13:41:45 +0300 Subject: [PATCH 149/548] README: add something about bearer tokens --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 79a387a..bd8105c 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,16 @@ object ID, like this: } ``` +#### Authentication + +You can always upload files to public containers (open for anyone to put +objects into), but for restricted containers you need to use bearer tokens +(which basically is an owner-signed ACL data, refer to NeoFS documentation for +more details). There are two options to pass them to gateway: + * "Authorization" header with "Bearer" type and base64-encoded token in + credentials field + * "__context_bearer_token_key" cookie with base64-encoded token contents + ### Metrics and Pprof If enabled, Prometheus metrics are available at `/metrics/` path and Pprof at From e47cb5f9c6fe2b6262da28f6502d990f13a0f0bd Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 14:39:53 +0300 Subject: [PATCH 150/548] workflows: add CodeQL workflow --- .github/workflows/codeql-analysis.yml | 67 +++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..942e14d --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,67 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '35 8 * * 1' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 0995dda59533b1217c11f633f90dc4d95e92fd83 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 14:40:32 +0300 Subject: [PATCH 151/548] .gitignore: extend with *~ --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 405b1dd..43e1681 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ temp /vendor/ .test.env +*~ *.log test.sh testfile From df43caa951410a7d5930e2c6dbd842fa2de6a10c Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 15:27:45 +0300 Subject: [PATCH 152/548] workflows: add build/lint workflows --- .github/workflows/builds.yml | 81 ++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/builds.yml diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml new file mode 100644 index 0000000..7556b77 --- /dev/null +++ b/.github/workflows/builds.yml @@ -0,0 +1,81 @@ +name: Builds and lints + +on: + pull_request: + branches: + - master + types: [opened, synchronize] + paths-ignore: + - '**/*.md' + workflow_dispatch: + +jobs: + lint: + name: Lint + runs-on: ubuntu-18.04 + + steps: + - uses: actions/checkout@v2 + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: latest + + build_cli: + name: Build CLI + runs-on: ubuntu-18.04 + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Restore Go modules from cache + uses: actions/cache@v2 + with: + path: /home/runner/go/pkg/mod + key: deps-${{ hashFiles('go.sum') }} + + - name: Update Go modules + run: make dep + + - name: Build CLI + run: make + + - name: Save binary + uses: actions/upload-artifact@v2 + with: + name: neofs-http-gw + path: bin/neofs-http-gw + + build_image: + needs: build_cli + name: Build Docker image + runs-on: ubuntu-18.04 + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Restore Go modules from cache + uses: actions/cache@v2 + with: + path: /home/runner/go/pkg/mod + key: deps-${{ hashFiles('go.sum') }} + + - name: Update Go modules + run: make dep + + - name: Build Docker image + run: make image From 6301a2c6b8bb9c44cbdc434a509c1dd35de1f4a4 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 15:34:14 +0300 Subject: [PATCH 153/548] workflow: add workflow to push images to Docker Hub --- .github/workflows/publish_to_dockerhub.yml | 82 ++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/publish_to_dockerhub.yml diff --git a/.github/workflows/publish_to_dockerhub.yml b/.github/workflows/publish_to_dockerhub.yml new file mode 100644 index 0000000..c6c773e --- /dev/null +++ b/.github/workflows/publish_to_dockerhub.yml @@ -0,0 +1,82 @@ +name: Push images to DockerHub + +# Controls when the action will run. +on: + push: + # Publish `master` as Docker `latest` and `git_revision` images. + branches: + - master + release: + # Publish released commit as Docker `latest` and `git_revision` images. + types: + - published + + # Allows to run this workflow manually from the Actions tab. + workflow_dispatch: + inputs: + ref: + description: 'Ref to build Docker image [default: latest master; examples: v0.92.0, 0a4ff9d3e4a9ab432fd5812eb18c98e03b5a7432]' + required: false + default: '' + push_image: + description: 'Push image to DockerHub [default: false; examples: true, false]' + required: false + default: 'false' + +# A workflow run. +jobs: + publish: + name: Publish image to DockerHub + runs-on: ubuntu-18.04 + steps: + - name: Checkout (manual run) + if: ${{ github.event_name == 'workflow_dispatch' }} + uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.ref }} + # Allows to fetch all history for all branches and tags. Need this for proper versioning. + fetch-depth: 0 + + - name: Checkout (automatical run) + if: ${{ github.event_name != 'workflow_dispatch' }} + uses: actions/checkout@v2 + with: + # Allows to fetch all history for all branches and tags. Need this for proper versioning. + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Restore go modules from cache + uses: actions/cache@v2 + with: + path: /home/runner/go/pkg/mod + key: deps-${{ hashFiles('go.sum') }} + + - name: Update Go modules + run: go mod download -json + + - name: Build image + run: make image + + - name: Build image with 'latest' tag + if: ${{ github.event_name == 'release' && github.event.release.target_commitish == 'master' }} + run: make image + env: + HUB_TAG:latest + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Push image to registry + if: ${{ github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.push_image == 'true') }} + run: make image-push + + - name: Push image with 'latest' tag to registry + if: ${{ github.event_name == 'release' && github.event.release.target_commitish == 'master' }} + run: make image-push-latest From 59f5f2da104f5fc632409b61253fc2f5ed9c6ca3 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 17:33:32 +0300 Subject: [PATCH 154/548] workflow: fix missing space The workflow is not valid. .github/workflows/publish_to_dockerhub.yml (Line: 68, Col: 11): Unexpected value 'HUB_TAG:latest' --- .github/workflows/publish_to_dockerhub.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_to_dockerhub.yml b/.github/workflows/publish_to_dockerhub.yml index c6c773e..c44d409 100644 --- a/.github/workflows/publish_to_dockerhub.yml +++ b/.github/workflows/publish_to_dockerhub.yml @@ -65,7 +65,7 @@ jobs: if: ${{ github.event_name == 'release' && github.event.release.target_commitish == 'master' }} run: make image env: - HUB_TAG:latest + HUB_TAG: latest - name: Login to DockerHub uses: docker/login-action@v1 From 3ee5ad52b66abc55e3092e9a7cbafada7b1b3a5e Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 17:38:48 +0300 Subject: [PATCH 155/548] workflows: use latest Ubuntu for linter Try to fix Error: NotifyContext not declared by package signal (typecheck) --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 7556b77..3d48f16 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -12,7 +12,7 @@ on: jobs: lint: name: Lint - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From 12859c2e5d95b8e5e7c807b16f284f60c16c48db Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 17:46:37 +0300 Subject: [PATCH 156/548] metrics: fix linter warning It's useless, but still we want to be green Error: missing cases in switch of type promhttp.HandlerErrorHandling: ContinueOnError (exhaustive) --- metrics.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metrics.go b/metrics.go index e888594..a827738 100644 --- a/metrics.go +++ b/metrics.go @@ -108,9 +108,10 @@ func metricsHandler(reg prometheus.Gatherer, opts promhttp.HandlerOpts) fasthttp case promhttp.HTTPErrorOnError: c.Error(err.Error(), fasthttp.StatusServiceUnavailable) return true + default: + // Do nothing in all other cases, including ContinueOnError. + return false } - // Do nothing in all other cases, including ContinueOnError. - return false } for _, mf := range mfs { From ebd3ad8842c1d1a27c0573d48a9861d73f148407 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 18:58:26 +0300 Subject: [PATCH 157/548] Makefile: add test and cover targets --- .gitignore | 3 +++ Makefile | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 43e1681..2726d2d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ testfile .neofs-cli.yml .cache + +coverage.txt +coverage.html diff --git a/Makefile b/Makefile index 77c7436..1797e7a 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ BINDIR = bin DIRS = $(BINDIR) BINS = "$(BINDIR)/neofs-http-gw" -.PHONY: help all dep clean fmts fmt imports test lint docker/lint +.PHONY: help all dep clean fmts fmt imports test cover lint docker/lint # Make all binaries all: $(BINS) @@ -43,6 +43,15 @@ dep: GO111MODULE=on \ go mod tidy -v && echo OK +# Run tests +test: + @go test ./... -cover + +# Run tests with race detection and produce coverage output +cover: + @go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic + @go tool cover -html=coverage.txt -o coverage.html + # Run all code formatters fmts: fmt imports From 93fcaffaaa80588830e9116e9f0bd96817ad848a Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 18:53:20 +0300 Subject: [PATCH 158/548] Makefile: add simple image-push target And use it in workflow. --- .github/workflows/publish_to_dockerhub.yml | 4 +++- Makefile | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_to_dockerhub.yml b/.github/workflows/publish_to_dockerhub.yml index c44d409..9e9d591 100644 --- a/.github/workflows/publish_to_dockerhub.yml +++ b/.github/workflows/publish_to_dockerhub.yml @@ -79,4 +79,6 @@ jobs: - name: Push image with 'latest' tag to registry if: ${{ github.event_name == 'release' && github.event.release.target_commitish == 'master' }} - run: make image-push-latest + run: make image-push + env: + HUB_TAG: latest diff --git a/Makefile b/Makefile index 77c7436..93a120d 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,11 @@ image: -f Dockerfile \ -t $(HUB_IMAGE):$(HUB_TAG) . +# Push Docker image to the hub +image-push: + @echo "⇒ Publish image" + @docker push $(HUB_IMAGE):$(HUB_TAG) + # Build dirty Docker image dirty-image: @echo "⇒ Build NeoFS HTTP Gateway dirty docker image " From 7ba5aed9cabebd767df03db67b9d9ea0c17f2fd0 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 19:02:30 +0300 Subject: [PATCH 159/548] tokens: fix tests Broken by afbb9d51f16413bc2e12aff6e8cb5133329f5c22 which changed some strings. --- tokens/bearer-token_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index c6ee703..f038014 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -82,17 +82,17 @@ func Test_fetchBearerToken(t *testing.T) { }{ {name: "empty"}, - {name: "bad base64 header", header: "WRONG BASE64", error: "could not fetch marshaled from base64"}, - {name: "bad base64 cookie", cookie: "WRONG BASE64", error: "could not fetch marshaled from base64"}, + {name: "bad base64 header", header: "WRONG BASE64", error: "can't base64-decode bearer token"}, + {name: "bad base64 cookie", cookie: "WRONG BASE64", error: "can't base64-decode bearer token"}, - {name: "header token unmarshal error", header: "dGVzdAo=", error: "could not unmarshal bearer token"}, - {name: "cookie token unmarshal error", cookie: "dGVzdAo=", error: "could not unmarshal bearer token"}, + {name: "header token unmarshal error", header: "dGVzdAo=", error: "can't unmarshal bearer token"}, + {name: "cookie token unmarshal error", cookie: "dGVzdAo=", error: "can't unmarshal bearer token"}, { name: "bad header and cookie", header: "WRONG BASE64", cookie: "dGVzdAo=", - error: "could not unmarshal bearer token", + error: "can't unmarshal bearer token", }, { From f05a6eda7da8aa251cfd1494adf2a670975e639f Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 19:08:57 +0300 Subject: [PATCH 160/548] workflows: add testing workflow --- .github/workflows/tests.yml | 75 +++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..e6ac620 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,75 @@ +name: Tests + +on: + pull_request: + branches: + - master + types: [opened, synchronize] + paths-ignore: + - '**/*.md' + workflow_dispatch: + +jobs: + cover: + name: Coverage + runs-on: ubuntu-18.04 + + env: + CGO_ENABLED: 0 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Restore Go modules from cache + uses: actions/cache@v2 + with: + path: /home/runner/go/pkg/mod + key: deps-${{ hashFiles('go.sum') }} + + - name: Update Go modules + run: make dep + + - name: Test and write coverage profile + run: make cover + + - name: Upload coverage results to Codecov + uses: codecov/codecov-action@v1 + with: + fail_ci_if_error: false + path_to_write_report: ./coverage.txt + verbose: true + + tests: + name: Tests + runs-on: ubuntu-18.04 + strategy: + matrix: + go_versions: [ '1.16' ] + fail-fast: false + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: '${{ matrix.go_versions }}' + + - name: Restore Go modules from cache + uses: actions/cache@v2 + with: + path: /home/runner/go/pkg/mod + key: deps-${{ hashFiles('go.sum') }} + + - name: Update Go modules + run: make dep + + - name: Run tests + run: make test From 25d273f88ee1b2db43bfa315edd211fb33db77f9 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 19:09:41 +0300 Subject: [PATCH 161/548] workflows: move linter to tests from builds --- .github/workflows/builds.yml | 13 +------------ .github/workflows/tests.yml | 11 +++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 3d48f16..078d286 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -1,4 +1,4 @@ -name: Builds and lints +name: Builds on: pull_request: @@ -10,17 +10,6 @@ on: workflow_dispatch: jobs: - lint: - name: Lint - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 - with: - version: latest - build_cli: name: Build CLI runs-on: ubuntu-18.04 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e6ac620..529f986 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,6 +10,17 @@ on: workflow_dispatch: jobs: + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: latest + cover: name: Coverage runs-on: ubuntu-18.04 From d5cdcb29c31010ff2952693490cff4dcf5f9553a Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 19:14:12 +0300 Subject: [PATCH 162/548] workflows: use Ubuntu 20.04 instead of 18.04 18.04 is a bit old. --- .github/workflows/builds.yml | 4 ++-- .github/workflows/publish_to_dockerhub.yml | 2 +- .github/workflows/tests.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 078d286..9532af8 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -12,7 +12,7 @@ on: jobs: build_cli: name: Build CLI - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 @@ -45,7 +45,7 @@ jobs: build_image: needs: build_cli name: Build Docker image - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/publish_to_dockerhub.yml b/.github/workflows/publish_to_dockerhub.yml index c44d409..4c6908d 100644 --- a/.github/workflows/publish_to_dockerhub.yml +++ b/.github/workflows/publish_to_dockerhub.yml @@ -27,7 +27,7 @@ on: jobs: publish: name: Publish image to DockerHub - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Checkout (manual run) if: ${{ github.event_name == 'workflow_dispatch' }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 529f986..09356a2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,7 +23,7 @@ jobs: cover: name: Coverage - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 env: CGO_ENABLED: 0 @@ -58,7 +58,7 @@ jobs: tests: name: Tests - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: matrix: go_versions: [ '1.16' ] From 7a7d27e8941116584c3744013bacf01664121cd3 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 19:43:06 +0300 Subject: [PATCH 163/548] workflows: use proper make dep for Docker image workflow `go mod download` changes go.sum making our images "-dirty". --- .github/workflows/publish_to_dockerhub.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_to_dockerhub.yml b/.github/workflows/publish_to_dockerhub.yml index bdf8e26..7a66b76 100644 --- a/.github/workflows/publish_to_dockerhub.yml +++ b/.github/workflows/publish_to_dockerhub.yml @@ -56,7 +56,7 @@ jobs: key: deps-${{ hashFiles('go.sum') }} - name: Update Go modules - run: go mod download -json + run: make dep - name: Build image run: make image From 9ab39fa22d44460064a7cf7159db0e3f285d1f1c Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 19:43:51 +0300 Subject: [PATCH 164/548] README: mention Docker images --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index bd8105c..4b0c71f 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,11 @@ NeoFS HTTP Protocol Gateway bridges NeoFS internal protocol and HTTP standard. Or you can call `make` to build it from the cloned repository (the binary will end up in `bin/neofs-http-gw`). +Or you can also use a [Docker +image](https://hub.docker.com/r/nspccdev/neofs-http-gw) provided for released +(and occasionaly unreleased) versions of gateway (`:latest` points to the +latest stable release). + ### Notable make targets ``` From ea66180012b00c794eeeceedbced5281b1f5ee15 Mon Sep 17 00:00:00 2001 From: Stanislav Bogatyrev Date: Fri, 30 Apr 2021 19:45:03 +0300 Subject: [PATCH 165/548] Improve first contribution experience Signed-off-by: Stanislav Bogatyrev --- .github/ISSUE_TEMPLATE/bug_report.md | 45 ++ .github/ISSUE_TEMPLATE/config.yml | 1 + .github/ISSUE_TEMPLATE/feature_request.md | 20 + .github/logo.svg | 129 +++++ .github/workflows/dco.yml | 21 + CONTRIBUTING.md | 156 +++++ LICENSE | 674 +++++++++++++++++++++ LICENSE.md | 675 ---------------------- README.md | 12 + 9 files changed, 1058 insertions(+), 675 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/logo.svg create mode 100644 .github/workflows/dco.yml create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE delete mode 100644 LICENSE.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..86b9297 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,45 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: community, triage +assignees: '' + +--- + + + +## Expected Behavior + + + +## Current Behavior + + + +## Possible Solution + + + +## Steps to Reproduce (for bugs) + + + +1. +2. +3. +4. + +## Context + + + +## Regression + + + +## Your Environment + +* Version used: +* Server setup and configuration: +* Operating System and version (`uname -a`): diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..3ba13e0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..4a7b218 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: community, triage +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/logo.svg b/.github/logo.svg new file mode 100644 index 0000000..b4da076 --- /dev/null +++ b/.github/logo.svg @@ -0,0 +1,129 @@ + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml new file mode 100644 index 0000000..40ed8fc --- /dev/null +++ b/.github/workflows/dco.yml @@ -0,0 +1,21 @@ +name: DCO check + +on: + pull_request: + branches: + - master + +jobs: + commits_check_job: + runs-on: ubuntu-latest + name: Commits Check + steps: + - name: Get PR Commits + id: 'get-pr-commits' + uses: tim-actions/get-pr-commits@master + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: DCO Check + uses: tim-actions/dco@master + with: + commits: ${{ steps.get-pr-commits.outputs.commits }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a1d0c71 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,156 @@ +# Contribution guide + +First, thank you for contributing! We love and encourage pull requests from +everyone. Please follow the guidelines: + +- Check the open [issues](https://github.com/nspcc-dev/neofs-http-gate/issues) and + [pull requests](https://github.com/nspcc-dev/neofs-http-gate/pulls) for existing + discussions. + +- Open an issue first, to discuss a new feature or enhancement. + +- Write tests, and make sure the test suite passes locally and on CI. + +- Open a pull request, and reference the relevant issue(s). + +- Make sure your commits are logically separated and have good comments + explaining the details of your change. + +- After receiving feedback, amend your commits or add new ones as + appropriate. + +- **Have fun!** + +## Development Workflow + +Start by forking the `neofs-http-gate` repository, make changes in a branch and then +send a pull request. We encourage pull requests to discuss code changes. Here +are the steps in details: + +### Set up your GitHub Repository +Fork [NeoFS HTTP Protocol Gateway +upstream](https://github.com/nspcc-dev/neofs-http-gate/fork) source repository +to your own personal repository. Copy the URL of your fork (you will need it for +the `git clone` command below). + +```sh +$ git clone https://github.com/nspcc-dev/neofs-http-gate +``` + +### Set up git remote as ``upstream`` +```sh +$ cd neofs-http-gate +$ git remote add upstream https://github.com/nspcc-dev/neofs-http-gate +$ git fetch upstream +$ git merge upstream/master +... +``` + +### Create your feature branch +Before making code changes, make sure you create a separate branch for these +changes. Maybe you will find it convenient to name branch in +`/-` format. + +``` +$ git checkout -b feature/123-something_awesome +``` + +### Test your changes +After your code changes, make sure + +- To add test cases for the new code. +- To run `make lint` +- To squash your commits into a single commit or a series of logically separated + commits run `git rebase -i`. It's okay to force update your pull request. +- To run `make test` and `make all` completes. + +### Commit changes +After verification, commit your changes. This is a [great +post](https://chris.beams.io/posts/git-commit/) on how to write useful commit +messages. Try following this template: + +``` +[#Issue] Summary + +Description + + + + +``` + +``` +$ git commit -am '[#123] Add some feature' +``` + +### Push to the branch +Push your locally committed changes to the remote origin (your fork) +``` +$ git push origin feature/123-something_awesome +``` + +### Create a Pull Request +Pull requests can be created via GitHub. Refer to [this +document](https://help.github.com/articles/creating-a-pull-request/) for +detailed steps on how to create a pull request. After a Pull Request gets peer +reviewed and approved, it will be merged. + +## DCO Sign off + +All authors to the project retain copyright to their work. However, to ensure +that they are only submitting work that they have rights to, we are requiring +everyone to acknowledge this by signing their work. + +Any copyright notices in this repository should specify the authors as "the +contributors". + +To sign your work, just add a line like this at the end of your commit message: + +``` +Signed-off-by: Samii Sakisaka +``` + +This can easily be done with the `--signoff` option to `git commit`. + +By doing this you state that you can certify the following (from [The Developer +Certificate of Origin](https://developercertificate.org/)): + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +1 Letterman Drive +Suite D4700 +San Francisco, CA, 94129 + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index 4d393d1..0000000 --- a/LICENSE.md +++ /dev/null @@ -1,675 +0,0 @@ -### GNU GENERAL PUBLIC LICENSE - -Version 3, 29 June 2007 - -Copyright (C) 2007 Free Software Foundation, Inc. - - -Everyone is permitted to copy and distribute verbatim copies of this -license document, but changing it is not allowed. - -### Preamble - -The GNU General Public License is a free, copyleft license for -software and other kinds of works. - -The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom -to share and change all versions of a program--to make sure it remains -free software for all its users. We, the Free Software Foundation, use -the GNU General Public License for most of our software; it applies -also to any other work released this way by its authors. You can apply -it to your programs, too. - -When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - -To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you -have certain responsibilities if you distribute copies of the -software, or if you modify it: responsibilities to respect the freedom -of others. - -For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - -Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - -For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - -Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the -manufacturer can do so. This is fundamentally incompatible with the -aim of protecting users' freedom to change the software. The -systematic pattern of such abuse occurs in the area of products for -individuals to use, which is precisely where it is most unacceptable. -Therefore, we have designed this version of the GPL to prohibit the -practice for those products. If such problems arise substantially in -other domains, we stand ready to extend this provision to those -domains in future versions of the GPL, as needed to protect the -freedom of users. - -Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish -to avoid the special danger that patents applied to a free program -could make it effectively proprietary. To prevent this, the GPL -assures that patents cannot be used to render the program non-free. - -The precise terms and conditions for copying, distribution and -modification follow. - -### TERMS AND CONDITIONS - -#### 0. Definitions. - -"This License" refers to version 3 of the GNU General Public License. - -"Copyright" also means copyright-like laws that apply to other kinds -of works, such as semiconductor masks. - -"The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - -To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of -an exact copy. The resulting work is called a "modified version" of -the earlier work or a work "based on" the earlier work. - -A "covered work" means either the unmodified Program or a work based -on the Program. - -To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - -To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user -through a computer network, with no transfer of a copy, is not -conveying. - -An interactive user interface displays "Appropriate Legal Notices" to -the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - -#### 1. Source Code. - -The "source code" for a work means the preferred form of the work for -making modifications to it. "Object code" means any non-source form of -a work. - -A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - -The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - -The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - -The Corresponding Source need not include anything that users can -regenerate automatically from other parts of the Corresponding Source. - -The Corresponding Source for a work in source code form is that same -work. - -#### 2. Basic Permissions. - -All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - -You may make, run and propagate covered works that you do not convey, -without conditions so long as your license otherwise remains in force. -You may convey covered works to others for the sole purpose of having -them make modifications exclusively for you, or provide you with -facilities for running those works, provided that you comply with the -terms of this License in conveying all material for which you do not -control copyright. Those thus making or running the covered works for -you must do so exclusively on your behalf, under your direction and -control, on terms that prohibit them from making any copies of your -copyrighted material outside their relationship with you. - -Conveying under any other circumstances is permitted solely under the -conditions stated below. Sublicensing is not allowed; section 10 makes -it unnecessary. - -#### 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - -No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - -When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such -circumvention is effected by exercising rights under this License with -respect to the covered work, and you disclaim any intention to limit -operation or modification of the work as a means of enforcing, against -the work's users, your or third parties' legal rights to forbid -circumvention of technological measures. - -#### 4. Conveying Verbatim Copies. - -You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - -You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - -#### 5. Conveying Modified Source Versions. - -You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these -conditions: - -- a) The work must carry prominent notices stating that you modified -it, and giving a relevant date. -- b) The work must carry prominent notices stating that it is -released under this License and any conditions added under -section 7. This requirement modifies the requirement in section 4 -to "keep intact all notices". -- c) You must license the entire work, as a whole, under this -License to anyone who comes into possession of a copy. This -License will therefore apply, along with any applicable section 7 -additional terms, to the whole of the work, and all its parts, -regardless of how they are packaged. This License gives no -permission to license the work in any other way, but it does not -invalidate such permission if you have separately received it. -- d) If the work has interactive user interfaces, each must display -Appropriate Legal Notices; however, if the Program has interactive -interfaces that do not display Appropriate Legal Notices, your -work need not make them do so. - -A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - -#### 6. Conveying Non-Source Forms. - -You may convey a covered work in object code form under the terms of -sections 4 and 5, provided that you also convey the machine-readable -Corresponding Source under the terms of this License, in one of these -ways: - -- a) Convey the object code in, or embodied in, a physical product -(including a physical distribution medium), accompanied by the -Corresponding Source fixed on a durable physical medium -customarily used for software interchange. -- b) Convey the object code in, or embodied in, a physical product -(including a physical distribution medium), accompanied by a -written offer, valid for at least three years and valid for as -long as you offer spare parts or customer support for that product -model, to give anyone who possesses the object code either (1) a -copy of the Corresponding Source for all the software in the -product that is covered by this License, on a durable physical -medium customarily used for software interchange, for a price no -more than your reasonable cost of physically performing this -conveying of source, or (2) access to copy the Corresponding -Source from a network server at no charge. -- c) Convey individual copies of the object code with a copy of the -written offer to provide the Corresponding Source. This -alternative is allowed only occasionally and noncommercially, and -only if you received the object code with such an offer, in accord -with subsection 6b. -- d) Convey the object code by offering access from a designated -place (gratis or for a charge), and offer equivalent access to the -Corresponding Source in the same way through the same place at no -further charge. You need not require recipients to copy the -Corresponding Source along with the object code. If the place to -copy the object code is a network server, the Corresponding Source -may be on a different server (operated by you or a third party) -that supports equivalent copying facilities, provided you maintain -clear directions next to the object code saying where to find the -Corresponding Source. Regardless of what server hosts the -Corresponding Source, you remain obligated to ensure that it is -available for as long as needed to satisfy these requirements. -- e) Convey the object code using peer-to-peer transmission, -provided you inform other peers where the object code and -Corresponding Source of the work are being offered to the general -public at no charge under subsection 6d. - -A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - -A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, -family, or household purposes, or (2) anything designed or sold for -incorporation into a dwelling. In determining whether a product is a -consumer product, doubtful cases shall be resolved in favor of -coverage. For a particular product received by a particular user, -"normally used" refers to a typical or common use of that class of -product, regardless of the status of the particular user or of the way -in which the particular user actually uses, or expects or is expected -to use, the product. A product is a consumer product regardless of -whether the product has substantial commercial, industrial or -non-consumer uses, unless such uses represent the only significant -mode of use of the product. - -"Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to -install and execute modified versions of a covered work in that User -Product from a modified version of its Corresponding Source. The -information must suffice to ensure that the continued functioning of -the modified object code is in no case prevented or interfered with -solely because modification has been made. - -If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - -The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or -updates for a work that has been modified or installed by the -recipient, or for the User Product in which it has been modified or -installed. Access to a network may be denied when the modification -itself materially and adversely affects the operation of the network -or violates the rules and protocols for communication across the -network. - -Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - -#### 7. Additional Terms. - -"Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - -When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - -Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders -of that material) supplement the terms of this License with terms: - -- a) Disclaiming warranty or limiting liability differently from the -terms of sections 15 and 16 of this License; or -- b) Requiring preservation of specified reasonable legal notices or -author attributions in that material or in the Appropriate Legal -Notices displayed by works containing it; or -- c) Prohibiting misrepresentation of the origin of that material, -or requiring that modified versions of such material be marked in -reasonable ways as different from the original version; or -- d) Limiting the use for publicity purposes of names of licensors -or authors of the material; or -- e) Declining to grant rights under trademark law for use of some -trade names, trademarks, or service marks; or -- f) Requiring indemnification of licensors and authors of that -material by anyone who conveys the material (or modified versions -of it) with contractual assumptions of liability to the recipient, -for any liability that these contractual assumptions directly -impose on those licensors and authors. - -All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - -If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - -Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; the -above requirements apply either way. - -#### 8. Termination. - -You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - -However, if you cease all violation of this License, then your license -from a particular copyright holder is reinstated (a) provisionally, -unless and until the copyright holder explicitly and finally -terminates your license, and (b) permanently, if the copyright holder -fails to notify you of the violation by some reasonable means prior to -60 days after the cessation. - -Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - -Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - -#### 9. Acceptance Not Required for Having Copies. - -You are not required to accept this License in order to receive or run -a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - -#### 10. Automatic Licensing of Downstream Recipients. - -Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - -An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - -You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - -#### 11. Patents. - -A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - -A contributor's "essential patent claims" are all patent claims owned -or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - -Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - -In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - -If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - -If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - -A patent license is "discriminatory" if it does not include within the -scope of its coverage, prohibits the exercise of, or is conditioned on -the non-exercise of one or more of the rights that are specifically -granted under this License. You may not convey a covered work if you -are a party to an arrangement with a third party that is in the -business of distributing software, under which you make payment to the -third party based on the extent of your activity of conveying the -work, and under which the third party grants, to any of the parties -who would receive the covered work from you, a discriminatory patent -license (a) in connection with copies of the covered work conveyed by -you (or copies made from those copies), or (b) primarily for and in -connection with specific products or compilations that contain the -covered work, unless you entered into that arrangement, or that patent -license was granted, prior to 28 March 2007. - -Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - -#### 12. No Surrender of Others' Freedom. - -If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under -this License and any other pertinent obligations, then as a -consequence you may not convey it at all. For example, if you agree to -terms that obligate you to collect a royalty for further conveying -from those to whom you convey the Program, the only way you could -satisfy both those terms and this License would be to refrain entirely -from conveying the Program. - -#### 13. Use with the GNU Affero General Public License. - -Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - -#### 14. Revised Versions of this License. - -The Free Software Foundation may publish revised and/or new versions -of the GNU General Public License from time to time. Such new versions -will be similar in spirit to the present version, but may differ in -detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies that a certain numbered version of the GNU General Public -License "or any later version" applies to it, you have the option of -following the terms and conditions either of that numbered version or -of any later version published by the Free Software Foundation. If the -Program does not specify a version number of the GNU General Public -License, you may choose any version ever published by the Free -Software Foundation. - -If the Program specifies that a proxy can decide which future versions -of the GNU General Public License can be used, that proxy's public -statement of acceptance of a version permanently authorizes you to -choose that version for the Program. - -Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - -#### 15. Disclaimer of Warranty. - -THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT -WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND -PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE -DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR -CORRECTION. - -#### 16. Limitation of Liability. - -IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR -CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES -ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT -NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR -LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM -TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER -PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -#### 17. Interpretation of Sections 15 and 16. - -If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - -END OF TERMS AND CONDITIONS - -### How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these -terms. - -To do so, attach the following notices to the program. It is safest to -attach them to the start of each source file to most effectively state -the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. - - -Copyright (C) - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -Also add information on how to contact you by electronic and paper -mail. - -If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) -This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands \`show w' and \`show c' should show the -appropriate parts of the General Public License. Of course, your -program's commands might be different; for a GUI interface, you would -use an "about box". - -You should also get your employer (if you work as a programmer) or -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. For more information on this, and how to apply and follow -the GNU GPL, see . - -The GNU General Public License does not permit incorporating your -program into proprietary programs. If your program is a subroutine -library, you may consider it more useful to permit linking proprietary -applications with the library. If this is what you want to do, use the -GNU Lesser General Public License instead of this License. But first, -please read . diff --git a/README.md b/README.md index bd8105c..366679a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,15 @@ +

+NeoFS +

+

+ NeoFS is a decentralized distributed object storage integrated with the NEO Blockchain. +

+ +--- +[![Report](https://goreportcard.com/badge/github.com/nspcc-dev/neofs-http-gate)](https://goreportcard.com/report/github.com/nspcc-dev/neofs-http-gate) +![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/nspcc-dev/neofs-http-gate?sort=semver) +![License](https://img.shields.io/github/license/nspcc-dev/neofs-http-gate.svg?style=popout) + # NeoFS HTTP Protocol Gateway NeoFS HTTP Protocol Gateway bridges NeoFS internal protocol and HTTP standard. From 42ef4f9bc4170ce21d25c64cc2da1dd8491a6969 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 21:18:59 +0300 Subject: [PATCH 166/548] CHANGELOG: add 0.15.0 release --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..863a0a8 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,37 @@ +# Changelog + +This document outlines major changes between releases. + +## 0.15.0 (30 Apr 2021) + +This is the first public release incorporating latest NeoFS protocol support +and fixing some bugs. + +New features: + * upload support (#14, #13, #29) + * ephemeral keys (#26) + * TLS server support (#28) + +Behavior changes: + * node weights can now be specified as simple numbers instead of percentages + and gateway will calculate the proportion automatically (#27) + * attributes are converted now to `X-Attribute-*` headers when retrieving + object from gate instead of `X-*` (#29) + +Improvements: + * better Makefile (#16, #24, #33) + * updated documentation (#16, #29) + * updated neofs-api-go to v1.25.0 (#17, #20) + * updated fasthttp to v1.23.0+ (#17, #29) + * refactoring, eliminating some dependencies (#20, #29) + +Bugs fixed: + * gateway attempted to work with no NeoFS peers configured (#29) + * some invalid headers could be sent for attributes using non-ASCII or + non-printable characters (#29) + +## Older versions + +Please refer to [Github +releases](https://github.com/nspcc-dev/neofs-http-gate/releases/) for older +releases. From 6d9bc994cd3b4d33e79496ea735c57df6cfbfd21 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 30 Apr 2021 21:46:14 +0300 Subject: [PATCH 167/548] CHANGELOG: add more references --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 863a0a8..5262bb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,8 @@ Behavior changes: object from gate instead of `X-*` (#29) Improvements: - * better Makefile (#16, #24, #33) - * updated documentation (#16, #29) + * better Makefile (#16, #24, #33, #34) + * updated documentation (#16, #29, #35, #36) * updated neofs-api-go to v1.25.0 (#17, #20) * updated fasthttp to v1.23.0+ (#17, #29) * refactoring, eliminating some dependencies (#20, #29) From 73bd26bc12332e5466c672bf1e2d0fb55a065146 Mon Sep 17 00:00:00 2001 From: Stanislav Bogatyrev Date: Tue, 4 May 2021 18:19:11 +0300 Subject: [PATCH 168/548] docs: Clarify auth scheme Signed-off-by: Stanislav Bogatyrev --- README.md | 54 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 4c5e48f..3ade29d 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,7 @@ NeoFS HTTP Protocol Gateway bridges NeoFS internal protocol and HTTP standard. Or you can call `make` to build it from the cloned repository (the binary will end up in `bin/neofs-http-gw`). -Or you can also use a [Docker -image](https://hub.docker.com/r/nspccdev/neofs-http-gw) provided for released -(and occasionaly unreleased) versions of gateway (`:latest` points to the -latest stable release). - -### Notable make targets +Notable make targets: ``` dep Check and ensure dependencies @@ -39,6 +34,11 @@ lint Run linters version Show current version ``` +Or you can also use a [Docker +image](https://hub.docker.com/r/nspccdev/neofs-http-gw) provided for released +(and occasionaly unreleased) versions of gateway (`:latest` points to the +latest stable release). + ## Execution HTTP gateway itself is not a NeoFS node, so to access NeoFS it uses node's @@ -54,13 +54,13 @@ $ neofs-http-gw -p 192.168.130.72:8080 $ HTTP_GW_PEERS_0_ADDRESS=192.168.130.72:8080 neofs-http-gw ``` -### Configuration +## Configuration In general, everything available as CLI parameter can also be specified via environment variables, so they're not specifically mentioned in most cases (see `--help` also). -#### Nodes and weights +### Nodes and weights You can specify multiple `-p` options to add more NeoFS nodes, this will make gateway spread requests equally among them (using weight 1 for every node): @@ -78,7 +78,7 @@ $ HTTP_GW_PEERS_0_ADDRESS=192.168.130.72:8080 HTTP_GW_PEERS_0_WEIGHT=9 \ This command will make gateway use 192.168.130.72 for 90% of requests and 192.168.130.71 for remaining 10%. -#### Keys +### Keys By default gateway autogenerates key pair it will use for NeoFS requests. If for some reason you need to have static keys you can pass them via `--key` @@ -89,7 +89,7 @@ string or (unencrypted) WIF string. Example: $ neofs-http-gw -p 192.168.130.72:8080 -k KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr ``` -#### Binding and TLS +### Binding and TLS Gateway binds to `0.0.0.0:8082` by default and you can change that with `--listen_address` option. @@ -107,7 +107,7 @@ $ neofs-http-gw -p 192.168.130.72:8080 --listen_address 192.168.130.130:443 \ --tls_key=key.pem --tls_certificate=cert.pem ``` -#### HTTP parameters +### HTTP parameters You can tune HTTP read and write buffer sizes as well as timeouts with `HTTP_GW_WEB_READ_BUFFER_SIZE`, `HTTP_GW_WEB_READ_TIMEOUT`, @@ -121,19 +121,19 @@ first and only then try sending it to NeoFS). `HTTP_GW_WEB_MAX_REQUEST_BODY_SIZE` controls maximum request body size limiting uploads to files slightly lower than this limit. -#### NeoFS parameters +### NeoFS parameters Gateway can automatically set timestamps for uploaded files based on local time source, use `HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP` environment variable to control this behavior. -#### Monitoring and metrics +### Monitoring and metrics Pprof and Prometheus are integrated into the gateway, but not enabled by default. To enable them use `--pprof` and `--metrics` flags or `HTTP_GW_PPROF`/`HTTP_GW_METRICS` environment variables. -#### Timeouts +### Timeouts You can tune gRPC interface parameters with `--connect_timeout` (for connection to node) and `--request_timeout` (for request processing over @@ -148,7 +148,7 @@ if needed. All timing options accept values with suffixes, so "15s" is 15 seconds and "2m" is 2 minutes. -#### Logging +### Logging `--verbose` flag enables gRPC logging and there is a number of environment variables to tune logging behavior: @@ -168,7 +168,7 @@ HTTP_GW_LOGGER_TRACE_LEVEL=string - Logger show trace on level This gateway intentionally provides limited feature set and doesn't try to substitute (or completely wrap) regular gRPC NeoFS interface. You can download and upload objects with it, but deleting, searching, managing ACLs, creating -containers and other activities are not supported and not planned to be +containers and other activities are not supported and not planned to be supported. ### Downloading @@ -180,7 +180,6 @@ requests to `/get/$CID/$OID` path, like this: ``` $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m8PtaoricLouCn5zE8hAFr3gZEBDCZFe9BEgVJTSocY - ``` There is also more complex interface provided for attribute-based downloads, @@ -285,13 +284,28 @@ object ID, like this: #### Authentication You can always upload files to public containers (open for anyone to put -objects into), but for restricted containers you need to use bearer tokens -(which basically is an owner-signed ACL data, refer to NeoFS documentation for -more details). There are two options to pass them to gateway: +objects into), but for restricted containers you need to explicitly allow PUT +operations for request signed with your HTTP Protocol Gateway keys. + +If your don't want to manage gateway's secret keys and adjust eACL rules when +gateway configuration changes (new gate, key rotation, etc) or you plan to use +public services, there is an option to let your application backend (or you) to +issue Bearer Tokens ans pass them from the client via gate down to NeoFS level +to grant access. + +NeoFS Bearer Token basically is a container owner-signed ACL data (refer to NeoFS +documentation for more details). There are two options to pass them to gateway: * "Authorization" header with "Bearer" type and base64-encoded token in credentials field * "__context_bearer_token_key" cookie with base64-encoded token contents +For example you have a mobile application frontend with a backend part storing +data in NeoFS. When user authorizes in mobile app, the backend issues a NeoFS +Bearer token and provides it to the frontend. Then the mobile app may generate +some data and upload it via any available NeoFS HTTP Protocol Gateway by adding +the corresponding header to the upload request. Accessing the ACL protected data +works the same way. + ### Metrics and Pprof If enabled, Prometheus metrics are available at `/metrics/` path and Pprof at From b5c08a8e73d1b6a1f5a0f46b1750a8f5baf34d78 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 12:39:01 +0300 Subject: [PATCH 169/548] misc: drop unused Debug variable Fix deadcode warning: misc.go:8:2 deadcode `Debug` is unused We have logging options and this one is not really needed. Signed-off-by: Roman Khimov --- Makefile | 4 +--- misc.go | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Makefile b/Makefile index bcee601..c4606e6 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --dirty --always) BUILD ?= $(shell date -u --iso=seconds) -DEBUG ?= false HUB_IMAGE ?= nspccdev/neofs-http-gw HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" @@ -24,8 +23,7 @@ $(BINS): $(DIRS) dep GO111MODULE=on \ go build -v -trimpath \ -ldflags "-X main.Version=$(VERSION) \ - -X main.Build=$(BUILD) \ - -X main.Debug=$(DEBUG)" \ + -X main.Build=$(BUILD)" \ -o $@ ./ $(DIRS): diff --git a/misc.go b/misc.go index d6fbe2d..1541334 100644 --- a/misc.go +++ b/misc.go @@ -5,5 +5,4 @@ const Prefix = "HTTP_GW" var ( Build = "now" Version = "dev" - Debug = "false" ) From 3173c70eb6d627415e580f883149b27cbfc82faf Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 12:41:25 +0300 Subject: [PATCH 170/548] settings: fix godot warnings Signed-off-by: Roman Khimov --- settings.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/settings.go b/settings.go index ca27a17..1239e65 100644 --- a/settings.go +++ b/settings.go @@ -30,12 +30,12 @@ const ( cfgTLSCertificate = "tls_certificate" cfgTLSKey = "tls_key" - // KeepAlive + // KeepAlive. cfgKeepaliveTime = "keepalive.time" cfgKeepaliveTimeout = "keepalive.timeout" cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream" - // Web + // Web. cfgWebReadBufferSize = "web.read_buffer_size" cfgWebWriteBufferSize = "web.write_buffer_size" cfgWebReadTimeout = "web.read_timeout" @@ -43,12 +43,12 @@ const ( cfgWebStreamRequestBody = "web.stream_request_body" cfgWebMaxRequestBodySize = "web.max_request_body_size" - // Timeouts + // Timeouts. cfgConTimeout = "connect_timeout" cfgReqTimeout = "request_timeout" cfgRebalance = "rebalance_timer" - // Logger: + // Logger. cfgLoggerLevel = "logger.level" cfgLoggerFormat = "logger.format" cfgLoggerTraceLevel = "logger.trace_level" @@ -57,18 +57,18 @@ const ( cfgLoggerSamplingInitial = "logger.sampling.initial" cfgLoggerSamplingThereafter = "logger.sampling.thereafter" - // Uploader Header + // Uploader Header. cfgUploaderHeaderEnableDefaultTimestamp = "upload_header.use_default_timestamp" - // Peers + // Peers. cfgPeers = "peers" - // Application + // Application. cfgApplicationName = "app.name" cfgApplicationVersion = "app.version" cfgApplicationBuildTime = "app.build_time" - // command line args + // Command line args. cmdHelp = "help" cmdVersion = "version" cmdVerbose = "verbose" From df3c87af79e71320fe57d33d2a829b8c0ebb66d5 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 15:22:03 +0300 Subject: [PATCH 171/548] *: fix all comment-related golint warnings Some of this code is going to be moved to SDK library, so it's important. Signed-off-by: Roman Khimov --- app.go | 4 ++++ connections/pool.go | 6 ++++++ connections/sampler.go | 7 ++++++- downloader/download.go | 4 ++++ global/context.go | 2 ++ logger/grpc.go | 18 ++++++++++++++++++ logger/option.go | 10 ++++++++++ logger/zap.go | 3 +++ misc.go | 6 +++++- neofs/client-plant.go | 16 ++++++++++++++++ neofs/credentials.go | 4 ++-- tokens/bearer-token.go | 6 ++++++ uploader/multipart.go | 2 ++ uploader/upload.go | 4 ++++ 14 files changed, 88 insertions(+), 4 deletions(-) diff --git a/app.go b/app.go index b42343b..bff85be 100644 --- a/app.go +++ b/app.go @@ -27,14 +27,17 @@ type ( webDone chan struct{} } + // App is an interface for the main gateway function. App interface { Wait() Serve(context.Context) } + // Option is an application option. Option func(a *app) ) +// WithLogger returns Option to set a specific logger. func WithLogger(l *zap.Logger) Option { return func(a *app) { if l == nil { @@ -44,6 +47,7 @@ func WithLogger(l *zap.Logger) Option { } } +// WithConfig returns Option to use specific Viper configuration. func WithConfig(c *viper.Viper) Option { return func(a *app) { if c == nil { diff --git a/connections/pool.go b/connections/pool.go index 5765d5f..4aae633 100644 --- a/connections/pool.go +++ b/connections/pool.go @@ -15,6 +15,7 @@ import ( "google.golang.org/grpc/keepalive" ) +// PoolBuilderOptions contains options used to build connection pool. type PoolBuilderOptions struct { Key *ecdsa.PrivateKey NodeConnectionTimeout time.Duration @@ -28,17 +29,21 @@ type PoolBuilderOptions struct { connections []*grpc.ClientConn } +// PoolBuilder is an interim structure used to collect node addresses/weights and +// build connection pool subsequently. type PoolBuilder struct { addresses []string weights []float64 } +// AddNode adds address/weight pair to node PoolBuilder list. func (pb *PoolBuilder) AddNode(address string, weight float64) *PoolBuilder { pb.addresses = append(pb.addresses, address) pb.weights = append(pb.weights, weight) return pb } +// Build creates new pool based on current PoolBuilder state and options. func (pb *PoolBuilder) Build(ctx context.Context, options *PoolBuilderOptions) (Pool, error) { if len(pb.addresses) == 0 { return nil, errors.New("no NeoFS peers configured") @@ -75,6 +80,7 @@ func (pb *PoolBuilder) Build(ctx context.Context, options *PoolBuilderOptions) ( return new(ctx, options) } +// Pool is an interface providing connection artifacts on request. type Pool interface { ConnectionArtifacts() (client.Client, *token.SessionToken, error) } diff --git a/connections/sampler.go b/connections/sampler.go index e6439af..caff5d6 100644 --- a/connections/sampler.go +++ b/connections/sampler.go @@ -2,13 +2,17 @@ package connections import "math/rand" -// See Vose's Alias Method (https://www.keithschwarz.com/darts-dice-coins/). +// Sampler implements weighted random number generation using Vose's Alias +// Method (https://www.keithschwarz.com/darts-dice-coins/). type Sampler struct { randomGenerator *rand.Rand probabilities []float64 alias []int } +// NewSampler creates new Sampler with a given set of probabilities using +// given source of randomness. Created Sampler will produce numbers from +// 0 to len(probabilities). func NewSampler(probabilities []float64, source rand.Source) *Sampler { sampler := &Sampler{} var ( @@ -53,6 +57,7 @@ func NewSampler(probabilities []float64, source rand.Source) *Sampler { return sampler } +// Next returns the next (not so) random number from Sampler. func (g *Sampler) Next() int { n := len(g.alias) i := g.randomGenerator.Intn(n) diff --git a/downloader/download.go b/downloader/download.go index 05a3bde..acaa7c9 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -165,11 +165,13 @@ func (o objectIDs) Slice() []string { return res } +// Downloader is a download request handler. type Downloader struct { log *zap.Logger plant neofs.ClientPlant } +// New creates an instance of Downloader using specified options. func New(ctx context.Context, log *zap.Logger, plant neofs.ClientPlant) (*Downloader, error) { var err error d := &Downloader{log: log, plant: plant} @@ -187,6 +189,7 @@ func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *requ } } +// DownloadByAddress handles download requests using simple cid/oid format. func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { var ( err error @@ -214,6 +217,7 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { d.newRequest(c, log).receiveFile(getOpts) } +// DownloadByAttribute handles attribute-based download requests. func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { var ( err error diff --git a/global/context.go b/global/context.go index 0fd29cf..8067ee8 100644 --- a/global/context.go +++ b/global/context.go @@ -12,6 +12,8 @@ var ( globalContextOnce sync.Once ) +// Context returns global context with initialized INT, TERM and HUP signal +// handlers set to notify this context. func Context() context.Context { globalContextOnce.Do(func() { globalContext, _ = signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) diff --git a/logger/grpc.go b/logger/grpc.go index 0b9635e..584bf25 100644 --- a/logger/grpc.go +++ b/logger/grpc.go @@ -12,12 +12,15 @@ type ( log *zap.SugaredLogger } + // Logger includes grpclog.LoggerV2 interface with an additional + // Println method. Logger interface { grpclog.LoggerV2 Println(v ...interface{}) } ) +// GRPC wraps given zap.Logger into grpclog.LoggerV2+ interface. func GRPC(l *zap.Logger) Logger { log := l.WithOptions( // skip gRPCLog + zapLogger in caller @@ -29,32 +32,47 @@ func GRPC(l *zap.Logger) Logger { } } +// Info implements grpclog.LoggerV2. func (z *zapLogger) Info(args ...interface{}) { z.log.Info(args...) } +// Infoln implements grpclog.LoggerV2. func (z *zapLogger) Infoln(args ...interface{}) { z.log.Info(args...) } +// Infof implements grpclog.LoggerV2. func (z *zapLogger) Infof(format string, args ...interface{}) { z.log.Infof(format, args...) } +// Println allows to print a line with info severity. func (z *zapLogger) Println(args ...interface{}) { z.log.Info(args...) } +// Printf implements grpclog.LoggerV2. func (z *zapLogger) Printf(format string, args ...interface{}) { z.log.Infof(format, args...) } +// Warning implements grpclog.LoggerV2. func (z *zapLogger) Warning(args ...interface{}) { z.log.Warn(args...) } +// Warningln implements grpclog.LoggerV2. func (z *zapLogger) Warningln(args ...interface{}) { z.log.Warn(args...) } +// Warningf implements grpclog.LoggerV2. func (z *zapLogger) Warningf(format string, args ...interface{}) { z.log.Warnf(format, args...) } +// Error implements grpclog.LoggerV2. func (z *zapLogger) Error(args ...interface{}) { z.log.Error(args...) } +// Errorln implements grpclog.LoggerV2. func (z *zapLogger) Errorln(args ...interface{}) { z.log.Error(args...) } +// Errorf implements grpclog.LoggerV2. func (z *zapLogger) Errorf(format string, args ...interface{}) { z.log.Errorf(format, args...) } +// Fatal implements grpclog.LoggerV2. func (z *zapLogger) Fatal(args ...interface{}) { z.log.Fatal(args...) } +// Fatalln implements grpclog.LoggerV2. func (z *zapLogger) Fatalln(args ...interface{}) { z.log.Fatal(args...) } +// Fatalf implements grpclog.LoggerV2. func (z *zapLogger) Fatalf(format string, args ...interface{}) { z.log.Fatalf(format, args...) } +// V implements grpclog.LoggerV2. func (z *zapLogger) V(int) bool { return z.Enabled(zapcore.DebugLevel) } diff --git a/logger/option.go b/logger/option.go index 23d1277..f617010 100644 --- a/logger/option.go +++ b/logger/option.go @@ -2,22 +2,32 @@ package logger import "go.uber.org/zap" +// WithSamplingInitial returns Option that sets sampling initial parameter. func WithSamplingInitial(v int) Option { return func(o *options) { o.SamplingInitial = v } } +// WithSamplingThereafter returns Option that sets sampling thereafter parameter. func WithSamplingThereafter(v int) Option { return func(o *options) { o.SamplingThereafter = v } } +// WithFormat returns Option that sets format parameter. func WithFormat(v string) Option { return func(o *options) { o.Format = v } } +// WithLevel returns Option that sets Level parameter. func WithLevel(v string) Option { return func(o *options) { o.Level = v } } +// WithTraceLevel returns Option that sets trace level parameter. func WithTraceLevel(v string) Option { return func(o *options) { o.TraceLevel = v } } +// WithoutDisclaimer returns Option that disables disclaimer. func WithoutDisclaimer() Option { return func(o *options) { o.NoDisclaimer = true } } +// WithoutCaller returns Option that disables caller printing. func WithoutCaller() Option { return func(o *options) { o.NoCaller = true } } +// WithAppName returns Option that sets application name. func WithAppName(v string) Option { return func(o *options) { o.AppName = v } } +// WithAppVersion returns Option that sets application version. func WithAppVersion(v string) Option { return func(o *options) { o.AppVersion = v } } +// WithZapOptions returns Option that sets zap logger options. func WithZapOptions(opts ...zap.Option) Option { return func(o *options) { o.Options = opts } } diff --git a/logger/zap.go b/logger/zap.go index 6228af8..5e9ee62 100644 --- a/logger/zap.go +++ b/logger/zap.go @@ -8,6 +8,7 @@ import ( ) type ( + // Option represents logger option setter. Option func(o *options) options struct { @@ -77,6 +78,8 @@ func defaults() *options { } } +// New returns new zap.Logger using all options specified and stdout used +// for output. func New(opts ...Option) (*zap.Logger, error) { o := defaults() c := zap.NewProductionConfig() diff --git a/misc.go b/misc.go index 1541334..d207131 100644 --- a/misc.go +++ b/misc.go @@ -1,8 +1,12 @@ package main +// Prefix is a prefix used for environment variables containing gateway +// configuration. const Prefix = "HTTP_GW" var ( - Build = "now" + // Build is a timestamp set during gateway build. + Build = "now" + // Version is gateway version. Version = "dev" ) diff --git a/neofs/client-plant.go b/neofs/client-plant.go index e2d6c5c..30a671b 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -13,12 +13,14 @@ import ( "github.com/nspcc-dev/neofs-http-gate/connections" ) +// BaseOptions represents basic NeoFS request options. type BaseOptions struct { Client client.Client SessionToken *token.SessionToken BearerToken *token.BearerToken } +// PutOptions represents NeoFS Put request options. type PutOptions struct { BaseOptions Attributes []*object.Attribute @@ -27,12 +29,14 @@ type PutOptions struct { Reader io.Reader } +// GetOptions represents NeoFS Get request options. type GetOptions struct { BaseOptions ObjectAddress *object.Address Writer io.Writer } +// SearchOptions represents NeoFS Search request options. type SearchOptions struct { BaseOptions ContainerID *container.ID @@ -42,11 +46,13 @@ type SearchOptions struct { } } +// DeleteOptions represents NeoFS Delete request options. type DeleteOptions struct { BaseOptions ObjectAddress *object.Address } +// ObjectClient wraps basic NeoFS requests. type ObjectClient interface { Put(context.Context, *PutOptions) (*object.Address, error) Get(context.Context, *GetOptions) (*object.Object, error) @@ -54,6 +60,8 @@ type ObjectClient interface { Delete(context.Context, *DeleteOptions) error } +// ClientPlant provides connections to NeoFS nodes from pool and allows to +// get local owner ID. type ClientPlant interface { ConnectionArtifacts() (client.Client, *token.SessionToken, error) Object() ObjectClient @@ -71,10 +79,12 @@ type neofsClientPlant struct { pool connections.Pool } +// ConnectionArtifacts returns connection from pool. func (cp *neofsClientPlant) ConnectionArtifacts() (client.Client, *token.SessionToken, error) { return cp.pool.ConnectionArtifacts() } +// Object returns ObjectClient instance from plant. func (cp *neofsClientPlant) Object() ObjectClient { return &neofsObjectClient{ key: cp.key, @@ -82,14 +92,17 @@ func (cp *neofsClientPlant) Object() ObjectClient { } } +// OwnerID returns plant's owner ID. func (cp *neofsClientPlant) OwnerID() *owner.ID { return cp.ownerID } +// NewClientPlant creates new ClientPlant from given context, pool and credentials. func NewClientPlant(ctx context.Context, pool connections.Pool, creds Credentials) (ClientPlant, error) { return &neofsClientPlant{key: creds.PrivateKey(), ownerID: creds.Owner(), pool: pool}, nil } +// Put does NeoFS Put request, returning new object address if successful. func (oc *neofsObjectClient) Put(ctx context.Context, options *PutOptions) (*object.Address, error) { var ( err error @@ -117,6 +130,7 @@ func (oc *neofsObjectClient) Put(ctx context.Context, options *PutOptions) (*obj return address, nil } +// Get does NeoFS Get request, returning an object received if successful. func (oc *neofsObjectClient) Get(ctx context.Context, options *GetOptions) (*object.Object, error) { var ( err error @@ -134,6 +148,7 @@ func (oc *neofsObjectClient) Get(ctx context.Context, options *GetOptions) (*obj return obj, err } +// Search does NeoFS Search request, returning object IDs if successful. func (oc *neofsObjectClient) Search(ctx context.Context, options *SearchOptions) ([]*object.ID, error) { sfs := object.NewSearchFilters() sfs.AddRootFilter() @@ -149,6 +164,7 @@ func (oc *neofsObjectClient) Search(ctx context.Context, options *SearchOptions) ) } +// Delete deletes NeoFS object. func (oc *neofsObjectClient) Delete(ctx context.Context, options *DeleteOptions) error { ops := new(client.DeleteObjectParams).WithAddress(options.ObjectAddress) err := options.Client.DeleteObject( diff --git a/neofs/credentials.go b/neofs/credentials.go index abc1ae9..c229468 100644 --- a/neofs/credentials.go +++ b/neofs/credentials.go @@ -24,8 +24,8 @@ type ( } ) -// New creates an instance of Credentials through string representation of secret. -// It allows passing WIF, path, hex-encoded and others. +// NewCredentials creates an instance of Credentials through string +// representation of secret. It allows passing WIF, path, hex-encoded and others. func NewCredentials(secret string) (Credentials, error) { key, err := crypto.LoadPrivateKey(secret) if err != nil { diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index 86e21f0..c7d16bc 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -26,6 +26,7 @@ const ( // return // } +// BearerTokenFromHeader extracts bearer token from Authorization request header. func BearerTokenFromHeader(h *fasthttp.RequestHeader) []byte { auth := h.Peek(fasthttp.HeaderAuthorization) if auth == nil || !bytes.HasPrefix(auth, []byte(bearerTokenHdr)) { @@ -37,6 +38,7 @@ func BearerTokenFromHeader(h *fasthttp.RequestHeader) []byte { return auth } +// BearerTokenFromCookie extracts bearer token from cookies. func BearerTokenFromCookie(h *fasthttp.RequestHeader) []byte { auth := h.Cookie(bearerTokenHdr) if len(auth) == 0 { @@ -46,6 +48,8 @@ func BearerTokenFromCookie(h *fasthttp.RequestHeader) []byte { return auth } +// StoreBearerToken extracts bearer token from header or cookie and stores +// it in the request context. func StoreBearerToken(ctx *fasthttp.RequestCtx) error { tkn, err := fetchBearerToken(ctx) if err != nil { @@ -56,6 +60,8 @@ func StoreBearerToken(ctx *fasthttp.RequestCtx) error { return nil } +// LoadBearerToken returns bearer token stored in context given (if it's +// present there). func LoadBearerToken(ctx context.Context) (*token.BearerToken, error) { if tkn, ok := ctx.Value(bearerTokenKey).(*token.BearerToken); ok && tkn != nil { return tkn, nil diff --git a/uploader/multipart.go b/uploader/multipart.go index 8abc6d0..c79ab94 100644 --- a/uploader/multipart.go +++ b/uploader/multipart.go @@ -7,6 +7,8 @@ import ( "go.uber.org/zap" ) +// MultipartFile provides standard ReadCloser interface and also allows one to +// get file name, it's used for multipart uploads. type MultipartFile interface { io.ReadCloser FileName() string diff --git a/uploader/upload.go b/uploader/upload.go index 02da473..719bee8 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -29,16 +29,20 @@ var putOptionsPool = sync.Pool{ }, } +// Uploader is an upload request handler. type Uploader struct { log *zap.Logger plant neofs.ClientPlant enableDefaultTimestamp bool } +// New creates a new Uploader using specified logger, connection pool and +// other options. func New(log *zap.Logger, plant neofs.ClientPlant, enableDefaultTimestamp bool) *Uploader { return &Uploader{log, plant, enableDefaultTimestamp} } +// Upload handles multipart upload request. func (u *Uploader) Upload(c *fasthttp.RequestCtx) { var ( err error From 9735f567af75e773e0c790dc7b1b43ca87b35b1e Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 15:22:40 +0300 Subject: [PATCH 172/548] golangci: use stricter settings Check test code, use more linters and reenable some disabled-by-default golint checks. Signed-off-by: Roman Khimov --- .golangci.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index ba892e7..79702d9 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,7 +7,7 @@ run: timeout: 5m # include test files or not, default is true - tests: false + tests: true # output configuration options output: @@ -32,16 +32,28 @@ linters: - golint # some default golangci-lint linters + - deadcode - errcheck - gosimple - ineffassign - staticcheck + - structcheck - typecheck + - unused + - varcheck # extra linters - exhaustive + - godot - gofmt - whitespace - goimports disable-all: true fast: false + +issues: + include: + - EXC0002 # should have a comment + - EXC0003 # test/Test ... consider calling this + - EXC0004 # govet + - EXC0005 # C-style breaks From 3bcd3a62897d4e8ce274a4887d09204679a5bb50 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 16:56:51 +0300 Subject: [PATCH 173/548] workflows: enable cgo for coverage test It's combined with `-race`, so we need cgo there: go test: -race requires cgo; enable cgo by setting CGO_ENABLED=1 make: *** [Makefile:50: cover] Error 2 Signed-off-by: Roman Khimov --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 09356a2..42f8b70 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-20.04 env: - CGO_ENABLED: 0 + CGO_ENABLED: 1 steps: - uses: actions/checkout@v2 with: From 685c229a97fc80db1a29be2b722a41de5a0ef223 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 7 May 2021 19:36:49 +0300 Subject: [PATCH 174/548] go.mod: update neofs-api-go to 1.26.0 Fix #39. Signed-off-by: Roman Khimov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8c2a78b..49c8739 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/fasthttp/router v1.3.5 github.com/mr-tron/base58 v1.1.3 // indirect github.com/nspcc-dev/neo-go v0.94.0 // indirect - github.com/nspcc-dev/neofs-api-go v1.25.0 + github.com/nspcc-dev/neofs-api-go v1.26.0 github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 diff --git a/go.sum b/go.sum index c0beb88..c44583d 100644 --- a/go.sum +++ b/go.sum @@ -314,8 +314,8 @@ github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5 github.com/nspcc-dev/neo-go v0.94.0 h1:2eafoyEnueqEMGDZF06HZJQN0MHgYxFzlcre5YUOP1M= github.com/nspcc-dev/neo-go v0.94.0/go.mod h1:IrBT/UG3/Slhqgjge8r6zni5tNRsWmwwG9OnVdASheI= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= -github.com/nspcc-dev/neofs-api-go v1.25.0 h1:hw1TTi3/3wCBB3KN6cycuM8Sn/MFnapSQptBx8KgDIY= -github.com/nspcc-dev/neofs-api-go v1.25.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-api-go v1.26.0 h1:kaQwCspb5k6xbxHBaEqh+0U5OUN0yowKP/QcPRQjCb0= +github.com/nspcc-dev/neofs-api-go v1.26.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= From ebb97a215d9e40de9e11840f952a2865fb7af46c Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Mon, 17 May 2021 17:30:16 +0300 Subject: [PATCH 175/548] Readme edits Add new info, new sections, examples Signed-off-by: Angira Kekteeva --- README.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3ade29d..abe4298 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,10 @@ can be done either via `-p` parameter or via `HTTP_GW_PEERS__ADDRESS` and `HTTP_GW_PEERS__WEIGHT` environment variables (the gate supports multiple NeoFS nodes with weighted load balancing). +If you're launching HTTP gateway in bundle with [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env), +you can get an IP address of the node in output of `make hosts` command +(with s0*.neofs.devenv name). + These two commands are functionally equivalent, they run the gate with one backend node (and otherwise default settings): ``` @@ -81,9 +85,13 @@ This command will make gateway use 192.168.130.72 for 90% of requests and ### Keys By default gateway autogenerates key pair it will use for NeoFS requests. If -for some reason you need to have static keys you can pass them via `--key` -parameter. The key can be a path to private key file (as raw bytes), a hex -string or (unencrypted) WIF string. Example: +for some reason you need to have static keys you can pass them via `--key` +(or `-k`) parameter. The key can be a path to private key file (as raw bytes), +a hex string or (unencrypted) WIF string: +``` +$ neofs-http-gw -p $NEOFS_NODE -k $KEY +``` +For example: ``` $ neofs-http-gw -p 192.168.130.72:8080 -k KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr @@ -171,26 +179,76 @@ and upload objects with it, but deleting, searching, managing ACLs, creating containers and other activities are not supported and not planned to be supported. +### Preparation + +Before uploading or downloading a file make sure you have a prepared container. +You can create it with instructions below. + +Also in case of downloading you need to have a file inside a container. + +#### Create a container + +You can create a container via [neofs-cli](https://github.com/nspcc-dev/neofs-node/releases): +``` +$ neofs-cli -r $NEOFS_NODE -k $KEY container create --policy $POLICY --basic-acl $ACL +``` +where `$KEY` can be a path to private key file (as raw bytes), a hex string or +(unencrypted) WIF string, +`$ACL` -- hex encoded basic ACL value or keywords 'private, 'public', 'readonly' and +`$POLICY` -- QL-encoded or JSON-encoded placement policy or path to file with it + +For example: +``` +$ neofs-cli -r 192.168.130.72:8080 -k 6PYLKJhiSub5imt6WCVy6Quxtd9xu176omev1vWYovzkAQCTSQabAAQXii container create --policy "REP 3" --basic-acl public --await +``` + +If you launched nodes via [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env) +you can get the key value from `wallets/wallet.json` or write the path to +the file `wallets/wallet.key`. + +#### Prepare a file in a container + +To create a file via [neofs-cli](https://github.com/nspcc-dev/neofs-node/releases), run a command below: +``` +$ neofs-cli -r $NEOFS_NODE -k $KEY object put --file $FILENAME --cid $CID +``` +where +`$KEY` -- the key, please read the information [above](#create-a-container), +`$CID` -- container ID. + +For example: +``` +$ neofs-cli -r 192.168.130.72:8080 -k 6PYLKJhiSub5imt6WCVy6Quxtd9xu176omev1vWYovzkAQCTSQabAAQXii object put --file cat.png --cid DPL2tpRiuDNmoTj5KZjD1nzDuCS8tVcxa7hsvSLDWpVM --attributes img_type=cat,my_attr=cute +``` + + ### Downloading #### Requests -Basic downloading involves container and object ID and is done via GET -requests to `/get/$CID/$OID` path, like this: +##### By IDs + +Basic downloading involves container ID and object ID and is done via GET +requests to `/get/$CID/$OID` path, where `$CID` is a container ID, +`$OID` is an object's (i.e. your file's) ID. + + For example: ``` $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m8PtaoricLouCn5zE8hAFr3gZEBDCZFe9BEgVJTSocY ``` +##### By attributes There is also more complex interface provided for attribute-based downloads, it's usually used to retrieve files by their names, but any other attribute can be used as well. The generic syntax for it looks like this: ```/get_by_attribute/$CID/$ATTRIBUTE_NAME/$ATTRIBUTE_VALUE``` -where `$CID` is a container ID, `$ATTRIBUTE_NAME` is the name of the attribute -we want to use and `ATTRIBUTE_VALUE` is the value of this attribute that the -target object should have. +where +`$CID` is a container ID, +`$ATTRIBUTE_NAME` is the name of the attribute we want to use, +`$ATTRIBUTE_VALUE` is the value of this attribute that the target object should have. If multiple objects have specified attribute with specified value, then the first one of them is returned (and you can't get others via this interface). From 33b1a28bf1594a9ce729670103eaa9cfc3eb8c7e Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Tue, 18 May 2021 14:18:50 +0300 Subject: [PATCH 176/548] Replace http-gate by http-gw in files Signed-off-by: Angira Kekteeva --- CHANGELOG.md | 2 +- CONTRIBUTING.md | 14 +++++++------- README.md | 8 ++++---- app.go | 12 ++++++------ downloader/download.go | 4 ++-- go.mod | 2 +- main.go | 4 ++-- neofs/client-plant.go | 2 +- settings.go | 2 +- uploader/upload.go | 4 ++-- 10 files changed, 27 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5262bb3..fc12fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,5 +33,5 @@ Bugs fixed: ## Older versions Please refer to [Github -releases](https://github.com/nspcc-dev/neofs-http-gate/releases/) for older +releases](https://github.com/nspcc-dev/neofs-http-gw/releases/) for older releases. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1d0c71..130a6e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,8 +3,8 @@ First, thank you for contributing! We love and encourage pull requests from everyone. Please follow the guidelines: -- Check the open [issues](https://github.com/nspcc-dev/neofs-http-gate/issues) and - [pull requests](https://github.com/nspcc-dev/neofs-http-gate/pulls) for existing +- Check the open [issues](https://github.com/nspcc-dev/neofs-http-gw/issues) and + [pull requests](https://github.com/nspcc-dev/neofs-http-gw/pulls) for existing discussions. - Open an issue first, to discuss a new feature or enhancement. @@ -23,24 +23,24 @@ everyone. Please follow the guidelines: ## Development Workflow -Start by forking the `neofs-http-gate` repository, make changes in a branch and then +Start by forking the `neofs-http-gw` repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: ### Set up your GitHub Repository Fork [NeoFS HTTP Protocol Gateway -upstream](https://github.com/nspcc-dev/neofs-http-gate/fork) source repository +upstream](https://github.com/nspcc-dev/neofs-http-gw/fork) source repository to your own personal repository. Copy the URL of your fork (you will need it for the `git clone` command below). ```sh -$ git clone https://github.com/nspcc-dev/neofs-http-gate +$ git clone https://github.com/nspcc-dev/neofs-http-gw ``` ### Set up git remote as ``upstream`` ```sh -$ cd neofs-http-gate -$ git remote add upstream https://github.com/nspcc-dev/neofs-http-gate +$ cd neofs-http-gw +$ git remote add upstream https://github.com/nspcc-dev/neofs-http-gw $ git fetch upstream $ git merge upstream/master ... diff --git a/README.md b/README.md index abe4298..ccbba33 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@

--- -[![Report](https://goreportcard.com/badge/github.com/nspcc-dev/neofs-http-gate)](https://goreportcard.com/report/github.com/nspcc-dev/neofs-http-gate) -![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/nspcc-dev/neofs-http-gate?sort=semver) -![License](https://img.shields.io/github/license/nspcc-dev/neofs-http-gate.svg?style=popout) +[![Report](https://goreportcard.com/badge/github.com/nspcc-dev/neofs-http-gw)](https://goreportcard.com/report/github.com/nspcc-dev/neofs-http-gw) +![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/nspcc-dev/neofs-http-gw?sort=semver) +![License](https://img.shields.io/github/license/nspcc-dev/neofs-http-gw.svg?style=popout) # NeoFS HTTP Protocol Gateway @@ -18,7 +18,7 @@ NeoFS HTTP Protocol Gateway bridges NeoFS internal protocol and HTTP standard. ## Installation -```go get -u github.com/nspcc-dev/neofs-http-gate``` +```go get -u github.com/nspcc-dev/neofs-http-gw``` Or you can call `make` to build it from the cloned repository (the binary will end up in `bin/neofs-http-gw`). diff --git a/app.go b/app.go index bff85be..841b305 100644 --- a/app.go +++ b/app.go @@ -6,11 +6,11 @@ import ( "strconv" "github.com/fasthttp/router" - "github.com/nspcc-dev/neofs-http-gate/connections" - "github.com/nspcc-dev/neofs-http-gate/downloader" - "github.com/nspcc-dev/neofs-http-gate/logger" - "github.com/nspcc-dev/neofs-http-gate/neofs" - "github.com/nspcc-dev/neofs-http-gate/uploader" + "github.com/nspcc-dev/neofs-http-gw/connections" + "github.com/nspcc-dev/neofs-http-gw/downloader" + "github.com/nspcc-dev/neofs-http-gw/logger" + "github.com/nspcc-dev/neofs-http-gw/neofs" + "github.com/nspcc-dev/neofs-http-gw/uploader" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -77,7 +77,7 @@ func newApp(ctx context.Context, opt ...Option) App { grpclog.SetLoggerV2(a.auxiliaryLog) } // -- setup FastHTTP server -- - a.webServer.Name = "neofs-http-gate" + a.webServer.Name = "neofs-http-gw" a.webServer.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) a.webServer.WriteBufferSize = a.cfg.GetInt(cfgWebWriteBufferSize) a.webServer.ReadTimeout = a.cfg.GetDuration(cfgWebReadTimeout) diff --git a/downloader/download.go b/downloader/download.go index acaa7c9..39ff41b 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -14,8 +14,8 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neofs-http-gate/neofs" - "github.com/nspcc-dev/neofs-http-gate/tokens" + "github.com/nspcc-dev/neofs-http-gw/neofs" + "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/valyala/fasthttp" "go.uber.org/zap" "google.golang.org/grpc/codes" diff --git a/go.mod b/go.mod index 49c8739..3cc7669 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/nspcc-dev/neofs-http-gate +module github.com/nspcc-dev/neofs-http-gw go 1.16 diff --git a/main.go b/main.go index 66f3f95..467f081 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,8 @@ package main import ( - "github.com/nspcc-dev/neofs-http-gate/global" - "github.com/nspcc-dev/neofs-http-gate/logger" + "github.com/nspcc-dev/neofs-http-gw/global" + "github.com/nspcc-dev/neofs-http-gw/logger" "github.com/spf13/viper" "go.uber.org/zap" ) diff --git a/neofs/client-plant.go b/neofs/client-plant.go index 30a671b..e13f29d 100644 --- a/neofs/client-plant.go +++ b/neofs/client-plant.go @@ -10,7 +10,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/token" - "github.com/nspcc-dev/neofs-http-gate/connections" + "github.com/nspcc-dev/neofs-http-gw/connections" ) // BaseOptions represents basic NeoFS request options. diff --git a/settings.go b/settings.go index 1239e65..1e5555c 100644 --- a/settings.go +++ b/settings.go @@ -119,7 +119,7 @@ func settings() *viper.Viper { peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") // set prefers: - v.Set(cfgApplicationName, "neofs-http-gate") + v.Set(cfgApplicationName, "neofs-http-gw") v.Set(cfgApplicationVersion, Version) // set defaults: diff --git a/uploader/upload.go b/uploader/upload.go index 719bee8..a9c874f 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -12,8 +12,8 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/token" - "github.com/nspcc-dev/neofs-http-gate/neofs" - "github.com/nspcc-dev/neofs-http-gate/tokens" + "github.com/nspcc-dev/neofs-http-gw/neofs" + "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/valyala/fasthttp" "go.uber.org/zap" ) From 51ede419af06372c545018d1fa6b1c49d6a080e6 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 24 May 2021 14:43:04 +0300 Subject: [PATCH 177/548] [#44] go.mod: update neofs-api-go to 1.26.1 Fix compatibility with the latest NeoFS nodes (0.20.0). Fixes #44. Signed-off-by: Roman Khimov --- go.mod | 3 +-- go.sum | 12 +++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 49c8739..76c74c8 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,7 @@ go 1.16 require ( github.com/fasthttp/router v1.3.5 github.com/mr-tron/base58 v1.1.3 // indirect - github.com/nspcc-dev/neo-go v0.94.0 // indirect - github.com/nspcc-dev/neofs-api-go v1.26.0 + github.com/nspcc-dev/neofs-api-go v1.26.1 github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 diff --git a/go.sum b/go.sum index c44583d..63228d1 100644 --- a/go.sum +++ b/go.sum @@ -21,9 +21,6 @@ github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1 github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= -github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= -github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -311,11 +308,11 @@ github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neo-go v0.94.0 h1:2eafoyEnueqEMGDZF06HZJQN0MHgYxFzlcre5YUOP1M= -github.com/nspcc-dev/neo-go v0.94.0/go.mod h1:IrBT/UG3/Slhqgjge8r6zni5tNRsWmwwG9OnVdASheI= +github.com/nspcc-dev/neo-go v0.95.0 h1:bttArYkIuhBJWSZsZ1xVW8MJsj5SvZwAhqVN3HZPNbo= +github.com/nspcc-dev/neo-go v0.95.0/go.mod h1:bW07ge1WFXsBgqrcPpLUr6OcyQxHqM26MZNesWMdH0c= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= -github.com/nspcc-dev/neofs-api-go v1.26.0 h1:kaQwCspb5k6xbxHBaEqh+0U5OUN0yowKP/QcPRQjCb0= -github.com/nspcc-dev/neofs-api-go v1.26.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= +github.com/nspcc-dev/neofs-api-go v1.26.1 h1:GMIuEB6Hv9IXP9SJd/1f8Df6gRriPkSplpmpJXgQ/1I= +github.com/nspcc-dev/neofs-api-go v1.26.1/go.mod h1:SHuH1Ba3U/h3j+8HHbb3Cns1LfMlEb88guWog9Qi68Y= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= @@ -454,6 +451,7 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= From a4ad52e181e3653ee43742e2beb8c594c487c8cc Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 24 May 2021 16:13:51 +0300 Subject: [PATCH 178/548] CHANGELOG: release 0.15.1 --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc12fa0..4a3c611 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ This document outlines major changes between releases. +## 0.15.1 (24 May 2021) + +This important release makes HTTP gateway compatible with NeoFS node version +0.20.0. + +Behavior changes: + * neofs-api-go was updated to 1.26.1, which contains some incompatible + changes in underlying components (#39, #44) + * `neofs-http-gw` is consistently used now for repository, binary and image + names (#43) + +Improvements: + * minor code cleanups based on stricter set of linters (#41) + * updated README (#42) + ## 0.15.0 (30 Apr 2021) This is the first public release incorporating latest NeoFS protocol support From 6c73296012b8031eb02d7303f5d547b6657a9728 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 25 May 2021 13:19:40 +0300 Subject: [PATCH 179/548] misc: drop Build variable Makes builds reproducible. Signed-off-by: Roman Khimov --- Makefile | 3 +-- misc.go | 2 -- settings.go | 20 +++++++++----------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index c4606e6..f152056 100644 --- a/Makefile +++ b/Makefile @@ -22,8 +22,7 @@ $(BINS): $(DIRS) dep CGO_ENABLED=0 \ GO111MODULE=on \ go build -v -trimpath \ - -ldflags "-X main.Version=$(VERSION) \ - -X main.Build=$(BUILD)" \ + -ldflags "-X main.Version=$(VERSION)" \ -o $@ ./ $(DIRS): diff --git a/misc.go b/misc.go index d207131..e82945a 100644 --- a/misc.go +++ b/misc.go @@ -5,8 +5,6 @@ package main const Prefix = "HTTP_GW" var ( - // Build is a timestamp set during gateway build. - Build = "now" // Version is gateway version. Version = "dev" ) diff --git a/settings.go b/settings.go index 1e5555c..4642cb0 100644 --- a/settings.go +++ b/settings.go @@ -64,9 +64,8 @@ const ( cfgPeers = "peers" // Application. - cfgApplicationName = "app.name" - cfgApplicationVersion = "app.version" - cfgApplicationBuildTime = "app.build_time" + cfgApplicationName = "app.name" + cfgApplicationVersion = "app.version" // Command line args. cmdHelp = "help" @@ -78,12 +77,11 @@ const ( ) var ignore = map[string]struct{}{ - cfgApplicationName: {}, - cfgApplicationVersion: {}, - cfgApplicationBuildTime: {}, - cfgPeers: {}, - cmdHelp: {}, - cmdVersion: {}, + cfgApplicationName: {}, + cfgApplicationVersion: {}, + cfgPeers: {}, + cmdHelp: {}, + cmdVersion: {}, } func (empty) Read([]byte) (int, error) { return 0, io.EOF } @@ -164,7 +162,7 @@ func settings() *viper.Viper { switch { case help != nil && *help: - fmt.Printf("NeoFS HTTP Gateway %s (%s)\n", Version, Build) + fmt.Printf("NeoFS HTTP Gateway %s\n", Version) flags.PrintDefaults() fmt.Println() @@ -191,7 +189,7 @@ func settings() *viper.Viper { os.Exit(0) case version != nil && *version: - fmt.Printf("NeoFS HTTP Gateway %s (%s)\n", Version, Build) + fmt.Printf("NeoFS HTTP Gateway %s\n", Version) os.Exit(0) } From 8ba5a2c92ae045c6f22688cc6436b38cfe5fab3a Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 25 May 2021 13:23:29 +0300 Subject: [PATCH 180/548] drop 'global' package It makes no sense as a package. Signed-off-by: Roman Khimov --- global/context.go | 22 ---------------------- main.go | 7 +++++-- 2 files changed, 5 insertions(+), 24 deletions(-) delete mode 100644 global/context.go diff --git a/global/context.go b/global/context.go deleted file mode 100644 index 8067ee8..0000000 --- a/global/context.go +++ /dev/null @@ -1,22 +0,0 @@ -package global - -import ( - "context" - "os/signal" - "sync" - "syscall" -) - -var ( - globalContext context.Context - globalContextOnce sync.Once -) - -// Context returns global context with initialized INT, TERM and HUP signal -// handlers set to notify this context. -func Context() context.Context { - globalContextOnce.Do(func() { - globalContext, _ = signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) - }) - return globalContext -} diff --git a/main.go b/main.go index 467f081..e89489b 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,10 @@ package main import ( - "github.com/nspcc-dev/neofs-http-gw/global" + "context" + "os/signal" + "syscall" + "github.com/nspcc-dev/neofs-http-gw/logger" "github.com/spf13/viper" "go.uber.org/zap" @@ -12,7 +15,7 @@ func main() { v = settings() l = newLogger(v) ) - globalContext := global.Context() + globalContext, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) app := newApp(globalContext, WithLogger(l), WithConfig(v)) go app.Serve(globalContext) app.Wait() From 82b2126bfdcf4551d563d4fc34eed02c55cc0de2 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Fri, 28 May 2021 11:57:28 +0300 Subject: [PATCH 181/548] [#46] *: Remove moved to sdk packages, refactoring Removed connections, logger, neofs because they were moved to sdk repo. Made changes in downloader, uploader, main.go and app.go via refactoring of neofs. Replaced dependencies to removed packages by sdk packages. Signed-off-by: Angira Kekteeva --- app.go | 10 +-- connections/pool.go | 160 ------------------------------------- connections/sampler.go | 81 ------------------- downloader/download.go | 83 +++++++++---------- go.mod | 2 +- go.sum | 4 + logger/grpc.go | 78 ------------------ logger/option.go | 33 -------- logger/zap.go | 134 ------------------------------- main.go | 2 +- neofs/client-plant.go | 177 ----------------------------------------- neofs/credentials.go | 78 ------------------ uploader/upload.go | 43 +++++----- 13 files changed, 73 insertions(+), 812 deletions(-) delete mode 100644 connections/pool.go delete mode 100644 connections/sampler.go delete mode 100644 logger/grpc.go delete mode 100644 logger/option.go delete mode 100644 logger/zap.go delete mode 100644 neofs/client-plant.go delete mode 100644 neofs/credentials.go diff --git a/app.go b/app.go index 841b305..e086ed2 100644 --- a/app.go +++ b/app.go @@ -6,11 +6,11 @@ import ( "strconv" "github.com/fasthttp/router" - "github.com/nspcc-dev/neofs-http-gw/connections" "github.com/nspcc-dev/neofs-http-gw/downloader" - "github.com/nspcc-dev/neofs-http-gw/logger" - "github.com/nspcc-dev/neofs-http-gw/neofs" "github.com/nspcc-dev/neofs-http-gw/uploader" + "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" + "github.com/nspcc-dev/neofs-sdk-go/pkg/neofs" + "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -99,7 +99,7 @@ func newApp(ctx context.Context, opt ...Option) App { if err != nil { a.log.Fatal("failed to get neofs credentials", zap.Error(err)) } - pb := new(connections.PoolBuilder) + pb := new(pool.Builder) for i := 0; ; i++ { address := a.cfg.GetString(cfgPeers + "." + strconv.Itoa(i) + ".address") weight := a.cfg.GetFloat64(cfgPeers + "." + strconv.Itoa(i) + ".weight") @@ -112,7 +112,7 @@ func newApp(ctx context.Context, opt ...Option) App { pb.AddNode(address, weight) a.log.Info("add connection", zap.String("address", address), zap.Float64("weight", weight)) } - opts := &connections.PoolBuilderOptions{ + opts := &pool.BuilderOptions{ Key: creds.PrivateKey(), NodeConnectionTimeout: a.cfg.GetDuration(cfgConTimeout), NodeRequestTimeout: a.cfg.GetDuration(cfgReqTimeout), diff --git a/connections/pool.go b/connections/pool.go deleted file mode 100644 index 4aae633..0000000 --- a/connections/pool.go +++ /dev/null @@ -1,160 +0,0 @@ -package connections - -import ( - "context" - "crypto/ecdsa" - "errors" - "fmt" - "math/rand" - "sync" - "time" - - "github.com/nspcc-dev/neofs-api-go/pkg/client" - "github.com/nspcc-dev/neofs-api-go/pkg/token" - "google.golang.org/grpc" - "google.golang.org/grpc/keepalive" -) - -// PoolBuilderOptions contains options used to build connection pool. -type PoolBuilderOptions struct { - Key *ecdsa.PrivateKey - NodeConnectionTimeout time.Duration - NodeRequestTimeout time.Duration - ClientRebalanceInterval time.Duration - KeepaliveTime time.Duration - KeepaliveTimeout time.Duration - KeepalivePermitWoStream bool - SessionExpirationEpoch uint64 - weights []float64 - connections []*grpc.ClientConn -} - -// PoolBuilder is an interim structure used to collect node addresses/weights and -// build connection pool subsequently. -type PoolBuilder struct { - addresses []string - weights []float64 -} - -// AddNode adds address/weight pair to node PoolBuilder list. -func (pb *PoolBuilder) AddNode(address string, weight float64) *PoolBuilder { - pb.addresses = append(pb.addresses, address) - pb.weights = append(pb.weights, weight) - return pb -} - -// Build creates new pool based on current PoolBuilder state and options. -func (pb *PoolBuilder) Build(ctx context.Context, options *PoolBuilderOptions) (Pool, error) { - if len(pb.addresses) == 0 { - return nil, errors.New("no NeoFS peers configured") - } - totalWeight := 0.0 - for _, w := range pb.weights { - totalWeight += w - } - for i, w := range pb.weights { - pb.weights[i] = w / totalWeight - } - var cons = make([]*grpc.ClientConn, len(pb.addresses)) - for i, address := range pb.addresses { - con, err := func() (*grpc.ClientConn, error) { - toctx, c := context.WithTimeout(ctx, options.NodeConnectionTimeout) - defer c() - return grpc.DialContext(toctx, address, - grpc.WithInsecure(), - grpc.WithBlock(), - grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: options.KeepaliveTime, - Timeout: options.KeepaliveTimeout, - PermitWithoutStream: options.KeepalivePermitWoStream, - }), - ) - }() - if err != nil { - return nil, err - } - cons[i] = con - } - options.weights = pb.weights - options.connections = cons - return new(ctx, options) -} - -// Pool is an interface providing connection artifacts on request. -type Pool interface { - ConnectionArtifacts() (client.Client, *token.SessionToken, error) -} - -type clientPack struct { - client client.Client - sessionToken *token.SessionToken - healthy bool -} - -type pool struct { - lock sync.RWMutex - sampler *Sampler - clientPacks []*clientPack -} - -func new(ctx context.Context, options *PoolBuilderOptions) (Pool, error) { - clientPacks := make([]*clientPack, len(options.weights)) - for i, con := range options.connections { - c, err := client.New(client.WithDefaultPrivateKey(options.Key), client.WithGRPCConnection(con)) - if err != nil { - return nil, err - } - st, err := c.CreateSession(ctx, options.SessionExpirationEpoch) - if err != nil { - address := "unknown" - if epi, err := c.EndpointInfo(ctx); err == nil { - address = epi.NodeInfo().Address() - } - return nil, fmt.Errorf("failed to create neofs session token for client %s: %w", address, err) - } - clientPacks[i] = &clientPack{client: c, sessionToken: st, healthy: true} - } - source := rand.NewSource(time.Now().UnixNano()) - sampler := NewSampler(options.weights, source) - pool := &pool{sampler: sampler, clientPacks: clientPacks} - go func() { - ticker := time.NewTimer(options.ClientRebalanceInterval) - for range ticker.C { - ok := true - for i, clientPack := range pool.clientPacks { - func() { - tctx, c := context.WithTimeout(ctx, options.NodeRequestTimeout) - defer c() - if _, err := clientPack.client.EndpointInfo(tctx); err != nil { - ok = false - } - pool.lock.Lock() - pool.clientPacks[i].healthy = ok - pool.lock.Unlock() - }() - } - ticker.Reset(options.ClientRebalanceInterval) - } - }() - return pool, nil -} - -func (p *pool) ConnectionArtifacts() (client.Client, *token.SessionToken, error) { - p.lock.RLock() - defer p.lock.RUnlock() - if len(p.clientPacks) == 1 { - cp := p.clientPacks[0] - if cp.healthy { - return cp.client, cp.sessionToken, nil - } - return nil, nil, errors.New("no healthy client") - } - attempts := 3 * len(p.clientPacks) - for k := 0; k < attempts; k++ { - i := p.sampler.Next() - if cp := p.clientPacks[i]; cp.healthy { - return cp.client, cp.sessionToken, nil - } - } - return nil, nil, errors.New("no healthy client") -} diff --git a/connections/sampler.go b/connections/sampler.go deleted file mode 100644 index caff5d6..0000000 --- a/connections/sampler.go +++ /dev/null @@ -1,81 +0,0 @@ -package connections - -import "math/rand" - -// Sampler implements weighted random number generation using Vose's Alias -// Method (https://www.keithschwarz.com/darts-dice-coins/). -type Sampler struct { - randomGenerator *rand.Rand - probabilities []float64 - alias []int -} - -// NewSampler creates new Sampler with a given set of probabilities using -// given source of randomness. Created Sampler will produce numbers from -// 0 to len(probabilities). -func NewSampler(probabilities []float64, source rand.Source) *Sampler { - sampler := &Sampler{} - var ( - small workList - large workList - ) - n := len(probabilities) - sampler.randomGenerator = rand.New(source) - sampler.probabilities = make([]float64, n) - sampler.alias = make([]int, n) - // Compute scaled probabilities. - p := make([]float64, n) - for i := 0; i < n; i++ { - p[i] = probabilities[i] * float64(n) - } - for i, pi := range p { - if pi < 1 { - small.add(i) - } else { - large.add(i) - } - } - for len(small) > 0 && len(large) > 0 { - l, g := small.remove(), large.remove() - sampler.probabilities[l] = p[l] - sampler.alias[l] = g - p[g] = p[g] + p[l] - 1 - if p[g] < 1 { - small.add(g) - } else { - large.add(g) - } - } - for len(large) > 0 { - g := large.remove() - sampler.probabilities[g] = 1 - } - for len(small) > 0 { - l := small.remove() - sampler.probabilities[l] = 1 - } - return sampler -} - -// Next returns the next (not so) random number from Sampler. -func (g *Sampler) Next() int { - n := len(g.alias) - i := g.randomGenerator.Intn(n) - if g.randomGenerator.Float64() < g.probabilities[i] { - return i - } - return g.alias[i] -} - -type workList []int - -func (wl *workList) add(e int) { - *wl = append(*wl, e) -} - -func (wl *workList) remove() int { - l := len(*wl) - 1 - n := (*wl)[l] - *wl = (*wl)[:l] - return n -} diff --git a/downloader/download.go b/downloader/download.go index 39ff41b..51baea2 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -12,30 +12,18 @@ import ( "sync" "time" + "github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neofs-http-gw/neofs" + "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-http-gw/tokens" + "github.com/nspcc-dev/neofs-sdk-go/pkg/neofs" "github.com/valyala/fasthttp" "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) -var ( - getOptionsPool = sync.Pool{ - New: func() interface{} { - return new(neofs.GetOptions) - }, - } - - searchOptionsPool = sync.Pool{ - New: func() interface{} { - return new(neofs.SearchOptions) - }, - } -) - type ( detector struct { io.Writer @@ -45,8 +33,7 @@ type ( request struct { *fasthttp.RequestCtx - log *zap.Logger - objectClient neofs.ObjectClient + log *zap.Logger } objectIDs []*object.ID @@ -85,12 +72,15 @@ func isValidValue(s string) bool { return true } -func (r *request) receiveFile(options *neofs.GetOptions) { +func (r *request) receiveFile(clnt client.Client, + sessionToken *token.SessionToken, + objectAddress *object.Address) { var ( err error dis = "inline" start = time.Now() filename string + obj *object.Object ) if err = tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) @@ -98,8 +88,15 @@ func (r *request) receiveFile(options *neofs.GetOptions) { return } writer := newDetector(r.Response.BodyWriter()) - options.Writer = writer - obj, err := r.objectClient.Get(r.RequestCtx, options) + options := new(client.GetObjectParams). + WithAddress(objectAddress). + WithPayloadWriter(writer) + + obj, err = clnt.GetObject( + r.RequestCtx, + options, + client.WithSession(sessionToken), + ) if err != nil { r.log.Error( "could not receive object", @@ -183,9 +180,8 @@ func New(ctx context.Context, log *zap.Logger, plant neofs.ClientPlant) (*Downlo func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { return &request{ - RequestCtx: ctx, - log: log, - objectClient: d.plant.Object(), + RequestCtx: ctx, + log: log, } } @@ -198,23 +194,22 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { oid, _ = c.UserValue("oid").(string) val = strings.Join([]string{cid, oid}, "/") log = d.log.With(zap.String("cid", cid), zap.String("oid", oid)) + conn client.Client + tkn *token.SessionToken ) if err = address.Parse(val); err != nil { log.Error("wrong object address", zap.Error(err)) c.Error("wrong object address", fasthttp.StatusBadRequest) return } - getOpts := getOptionsPool.Get().(*neofs.GetOptions) - defer getOptionsPool.Put(getOpts) - getOpts.Client, getOpts.SessionToken, err = d.plant.ConnectionArtifacts() + + conn, tkn, err = d.plant.ConnectionArtifacts() if err != nil { log.Error("failed to get neofs connection artifacts", zap.Error(err)) c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) return } - getOpts.ObjectAddress = address - getOpts.Writer = nil - d.newRequest(c, log).receiveFile(getOpts) + d.newRequest(c, log).receiveFile(conn, tkn, address) } // DownloadByAttribute handles attribute-based download requests. @@ -225,6 +220,9 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { key, _ = c.UserValue("attr_key").(string) val, _ = c.UserValue("attr_val").(string) log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) + ids []*object.ID + conn client.Client + tkn *token.SessionToken ) cid := container.NewID() if err = cid.Parse(scid); err != nil { @@ -232,20 +230,20 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { c.Error("wrong container id", fasthttp.StatusBadRequest) return } - searchOpts := searchOptionsPool.Get().(*neofs.SearchOptions) - defer searchOptionsPool.Put(searchOpts) - searchOpts.Client, searchOpts.SessionToken, err = d.plant.ConnectionArtifacts() + + conn, tkn, err = d.plant.ConnectionArtifacts() if err != nil { log.Error("failed to get neofs connection artifacts", zap.Error(err)) c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) return } - searchOpts.BearerToken = nil - searchOpts.ContainerID = cid - searchOpts.Attribute.Key = key - searchOpts.Attribute.Value = val - var ids []*object.ID - if ids, err = d.plant.Object().Search(c, searchOpts); err != nil { + + options := object.NewSearchFilters() + options.AddRootFilter() + options.AddFilter(key, val, object.MatchStringEqual) + + sops := new(client.SearchObjectParams).WithContainerID(cid).WithSearchFilters(options) + if ids, err = conn.SearchObject(c, sops, client.WithSession(tkn)); err != nil { log.Error("something went wrong", zap.Error(err)) c.Error("something went wrong", fasthttp.StatusBadRequest) return @@ -262,15 +260,12 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { address := object.NewAddress() address.SetContainerID(cid) address.SetObjectID(ids[0]) - getOpts := getOptionsPool.Get().(*neofs.GetOptions) - defer getOptionsPool.Put(getOpts) - getOpts.Client, getOpts.SessionToken, err = d.plant.ConnectionArtifacts() + + conn, tkn, err = d.plant.ConnectionArtifacts() if err != nil { log.Error("failed to get neofs connection artifacts", zap.Error(err)) c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) return } - getOpts.ObjectAddress = address - getOpts.Writer = nil - d.newRequest(c, log).receiveFile(getOpts) + d.newRequest(c, log).receiveFile(conn, tkn, address) } diff --git a/go.mod b/go.mod index 2919778..059c184 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/fasthttp/router v1.3.5 github.com/mr-tron/base58 v1.1.3 // indirect github.com/nspcc-dev/neofs-api-go v1.26.1 - github.com/nspcc-dev/neofs-crypto v0.3.0 + github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210527182636-cbfc17a1a9a2 github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 63228d1..6459a87 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzg github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/alecthomas/participle v0.7.1/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY= +github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -317,6 +319,8 @@ github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9K github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210527182636-cbfc17a1a9a2 h1:z8xtKILKi+Dolk3VAyCaFPMroFnT+x8qTqMT/zBRqIc= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210527182636-cbfc17a1a9a2/go.mod h1:QZE7VaNQRyNFS+3gsrNEQEiLe+d6AR6EteX1M9geh6A= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/logger/grpc.go b/logger/grpc.go deleted file mode 100644 index 584bf25..0000000 --- a/logger/grpc.go +++ /dev/null @@ -1,78 +0,0 @@ -package logger - -import ( - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "google.golang.org/grpc/grpclog" -) - -type ( - zapLogger struct { - zapcore.Core - log *zap.SugaredLogger - } - - // Logger includes grpclog.LoggerV2 interface with an additional - // Println method. - Logger interface { - grpclog.LoggerV2 - Println(v ...interface{}) - } -) - -// GRPC wraps given zap.Logger into grpclog.LoggerV2+ interface. -func GRPC(l *zap.Logger) Logger { - log := l.WithOptions( - // skip gRPCLog + zapLogger in caller - zap.AddCallerSkip(2)) - - return &zapLogger{ - Core: log.Core(), - log: log.Sugar(), - } -} - -// Info implements grpclog.LoggerV2. -func (z *zapLogger) Info(args ...interface{}) { z.log.Info(args...) } - -// Infoln implements grpclog.LoggerV2. -func (z *zapLogger) Infoln(args ...interface{}) { z.log.Info(args...) } - -// Infof implements grpclog.LoggerV2. -func (z *zapLogger) Infof(format string, args ...interface{}) { z.log.Infof(format, args...) } - -// Println allows to print a line with info severity. -func (z *zapLogger) Println(args ...interface{}) { z.log.Info(args...) } - -// Printf implements grpclog.LoggerV2. -func (z *zapLogger) Printf(format string, args ...interface{}) { z.log.Infof(format, args...) } - -// Warning implements grpclog.LoggerV2. -func (z *zapLogger) Warning(args ...interface{}) { z.log.Warn(args...) } - -// Warningln implements grpclog.LoggerV2. -func (z *zapLogger) Warningln(args ...interface{}) { z.log.Warn(args...) } - -// Warningf implements grpclog.LoggerV2. -func (z *zapLogger) Warningf(format string, args ...interface{}) { z.log.Warnf(format, args...) } - -// Error implements grpclog.LoggerV2. -func (z *zapLogger) Error(args ...interface{}) { z.log.Error(args...) } - -// Errorln implements grpclog.LoggerV2. -func (z *zapLogger) Errorln(args ...interface{}) { z.log.Error(args...) } - -// Errorf implements grpclog.LoggerV2. -func (z *zapLogger) Errorf(format string, args ...interface{}) { z.log.Errorf(format, args...) } - -// Fatal implements grpclog.LoggerV2. -func (z *zapLogger) Fatal(args ...interface{}) { z.log.Fatal(args...) } - -// Fatalln implements grpclog.LoggerV2. -func (z *zapLogger) Fatalln(args ...interface{}) { z.log.Fatal(args...) } - -// Fatalf implements grpclog.LoggerV2. -func (z *zapLogger) Fatalf(format string, args ...interface{}) { z.log.Fatalf(format, args...) } - -// V implements grpclog.LoggerV2. -func (z *zapLogger) V(int) bool { return z.Enabled(zapcore.DebugLevel) } diff --git a/logger/option.go b/logger/option.go deleted file mode 100644 index f617010..0000000 --- a/logger/option.go +++ /dev/null @@ -1,33 +0,0 @@ -package logger - -import "go.uber.org/zap" - -// WithSamplingInitial returns Option that sets sampling initial parameter. -func WithSamplingInitial(v int) Option { return func(o *options) { o.SamplingInitial = v } } - -// WithSamplingThereafter returns Option that sets sampling thereafter parameter. -func WithSamplingThereafter(v int) Option { return func(o *options) { o.SamplingThereafter = v } } - -// WithFormat returns Option that sets format parameter. -func WithFormat(v string) Option { return func(o *options) { o.Format = v } } - -// WithLevel returns Option that sets Level parameter. -func WithLevel(v string) Option { return func(o *options) { o.Level = v } } - -// WithTraceLevel returns Option that sets trace level parameter. -func WithTraceLevel(v string) Option { return func(o *options) { o.TraceLevel = v } } - -// WithoutDisclaimer returns Option that disables disclaimer. -func WithoutDisclaimer() Option { return func(o *options) { o.NoDisclaimer = true } } - -// WithoutCaller returns Option that disables caller printing. -func WithoutCaller() Option { return func(o *options) { o.NoCaller = true } } - -// WithAppName returns Option that sets application name. -func WithAppName(v string) Option { return func(o *options) { o.AppName = v } } - -// WithAppVersion returns Option that sets application version. -func WithAppVersion(v string) Option { return func(o *options) { o.AppVersion = v } } - -// WithZapOptions returns Option that sets zap logger options. -func WithZapOptions(opts ...zap.Option) Option { return func(o *options) { o.Options = opts } } diff --git a/logger/zap.go b/logger/zap.go deleted file mode 100644 index 5e9ee62..0000000 --- a/logger/zap.go +++ /dev/null @@ -1,134 +0,0 @@ -package logger - -import ( - "strings" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -type ( - // Option represents logger option setter. - Option func(o *options) - - options struct { - Options []zap.Option - - SamplingInitial int - SamplingThereafter int - - Format string - Level string - TraceLevel string - - NoCaller bool - NoDisclaimer bool - - AppName string - AppVersion string - } -) - -const ( - formatJSON = "json" - formatConsole = "console" - - defaultSamplingInitial = 100 - defaultSamplingThereafter = 100 - - lvlInfo = "info" - lvlWarn = "warn" - lvlDebug = "debug" - lvlError = "error" - lvlFatal = "fatal" - lvlPanic = "panic" -) - -func safeLevel(lvl string) zap.AtomicLevel { - switch strings.ToLower(lvl) { - case lvlDebug: - return zap.NewAtomicLevelAt(zap.DebugLevel) - case lvlWarn: - return zap.NewAtomicLevelAt(zap.WarnLevel) - case lvlError: - return zap.NewAtomicLevelAt(zap.ErrorLevel) - case lvlFatal: - return zap.NewAtomicLevelAt(zap.FatalLevel) - case lvlPanic: - return zap.NewAtomicLevelAt(zap.PanicLevel) - default: - return zap.NewAtomicLevelAt(zap.InfoLevel) - } -} - -func defaults() *options { - return &options{ - SamplingInitial: defaultSamplingInitial, - SamplingThereafter: defaultSamplingThereafter, - - Format: formatConsole, - Level: lvlDebug, - TraceLevel: lvlInfo, - - NoCaller: false, - NoDisclaimer: false, - - AppName: "", - AppVersion: "", - } -} - -// New returns new zap.Logger using all options specified and stdout used -// for output. -func New(opts ...Option) (*zap.Logger, error) { - o := defaults() - c := zap.NewProductionConfig() - - c.OutputPaths = []string{"stdout"} - c.ErrorOutputPaths = []string{"stdout"} - - for _, opt := range opts { - opt(o) - } - - // set sampling - c.Sampling = &zap.SamplingConfig{ - Initial: o.SamplingInitial, - Thereafter: o.SamplingThereafter, - } - - // logger level - c.Level = safeLevel(o.Level) - traceLvl := safeLevel(o.TraceLevel) - - // logger format - switch f := o.Format; strings.ToLower(f) { - case formatConsole: - c.Encoding = formatConsole - default: - c.Encoding = formatJSON - } - - // logger time - c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - - if o.NoCaller { - c.EncoderConfig.EncodeCaller = nil - } - - // enable trace only for current log-level - o.Options = append(o.Options, zap.AddStacktrace(traceLvl)) - - l, err := c.Build(o.Options...) - if err != nil { - return nil, err - } - - if o.NoDisclaimer { - return l, nil - } - - return l.With( - zap.String("app_name", o.AppName), - zap.String("app_version", o.AppVersion)), nil -} diff --git a/main.go b/main.go index e89489b..830a26f 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,7 @@ import ( "os/signal" "syscall" - "github.com/nspcc-dev/neofs-http-gw/logger" + "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" "github.com/spf13/viper" "go.uber.org/zap" ) diff --git a/neofs/client-plant.go b/neofs/client-plant.go deleted file mode 100644 index e13f29d..0000000 --- a/neofs/client-plant.go +++ /dev/null @@ -1,177 +0,0 @@ -package neofs - -import ( - "context" - "crypto/ecdsa" - "io" - - "github.com/nspcc-dev/neofs-api-go/pkg/client" - "github.com/nspcc-dev/neofs-api-go/pkg/container" - "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neofs-api-go/pkg/owner" - "github.com/nspcc-dev/neofs-api-go/pkg/token" - "github.com/nspcc-dev/neofs-http-gw/connections" -) - -// BaseOptions represents basic NeoFS request options. -type BaseOptions struct { - Client client.Client - SessionToken *token.SessionToken - BearerToken *token.BearerToken -} - -// PutOptions represents NeoFS Put request options. -type PutOptions struct { - BaseOptions - Attributes []*object.Attribute - ContainerID *container.ID - OwnerID *owner.ID - Reader io.Reader -} - -// GetOptions represents NeoFS Get request options. -type GetOptions struct { - BaseOptions - ObjectAddress *object.Address - Writer io.Writer -} - -// SearchOptions represents NeoFS Search request options. -type SearchOptions struct { - BaseOptions - ContainerID *container.ID - Attribute struct { - Key string - Value string - } -} - -// DeleteOptions represents NeoFS Delete request options. -type DeleteOptions struct { - BaseOptions - ObjectAddress *object.Address -} - -// ObjectClient wraps basic NeoFS requests. -type ObjectClient interface { - Put(context.Context, *PutOptions) (*object.Address, error) - Get(context.Context, *GetOptions) (*object.Object, error) - Search(context.Context, *SearchOptions) ([]*object.ID, error) - Delete(context.Context, *DeleteOptions) error -} - -// ClientPlant provides connections to NeoFS nodes from pool and allows to -// get local owner ID. -type ClientPlant interface { - ConnectionArtifacts() (client.Client, *token.SessionToken, error) - Object() ObjectClient - OwnerID() *owner.ID -} - -type neofsObjectClient struct { - key *ecdsa.PrivateKey - pool connections.Pool -} - -type neofsClientPlant struct { - key *ecdsa.PrivateKey - ownerID *owner.ID - pool connections.Pool -} - -// ConnectionArtifacts returns connection from pool. -func (cp *neofsClientPlant) ConnectionArtifacts() (client.Client, *token.SessionToken, error) { - return cp.pool.ConnectionArtifacts() -} - -// Object returns ObjectClient instance from plant. -func (cp *neofsClientPlant) Object() ObjectClient { - return &neofsObjectClient{ - key: cp.key, - pool: cp.pool, - } -} - -// OwnerID returns plant's owner ID. -func (cp *neofsClientPlant) OwnerID() *owner.ID { - return cp.ownerID -} - -// NewClientPlant creates new ClientPlant from given context, pool and credentials. -func NewClientPlant(ctx context.Context, pool connections.Pool, creds Credentials) (ClientPlant, error) { - return &neofsClientPlant{key: creds.PrivateKey(), ownerID: creds.Owner(), pool: pool}, nil -} - -// Put does NeoFS Put request, returning new object address if successful. -func (oc *neofsObjectClient) Put(ctx context.Context, options *PutOptions) (*object.Address, error) { - var ( - err error - objectID *object.ID - ) - address := object.NewAddress() - rawObject := object.NewRaw() - rawObject.SetContainerID(options.ContainerID) - rawObject.SetOwnerID(options.OwnerID) - rawObject.SetAttributes(options.Attributes...) - ops := new(client.PutObjectParams). - WithObject(rawObject.Object()). - WithPayloadReader(options.Reader) - objectID, err = options.Client.PutObject( - ctx, - ops, - client.WithSession(options.SessionToken), - client.WithBearer(options.BearerToken), - ) - if err != nil { - return nil, err - } - address.SetObjectID(objectID) - address.SetContainerID(options.ContainerID) - return address, nil -} - -// Get does NeoFS Get request, returning an object received if successful. -func (oc *neofsObjectClient) Get(ctx context.Context, options *GetOptions) (*object.Object, error) { - var ( - err error - obj *object.Object - ) - ops := new(client.GetObjectParams). - WithAddress(options.ObjectAddress). - WithPayloadWriter(options.Writer) - obj, err = options.Client.GetObject( - ctx, - ops, - client.WithSession(options.SessionToken), - client.WithBearer(options.BearerToken), - ) - return obj, err -} - -// Search does NeoFS Search request, returning object IDs if successful. -func (oc *neofsObjectClient) Search(ctx context.Context, options *SearchOptions) ([]*object.ID, error) { - sfs := object.NewSearchFilters() - sfs.AddRootFilter() - sfs.AddFilter(options.Attribute.Key, options.Attribute.Value, object.MatchStringEqual) - sops := new(client.SearchObjectParams) - sops.WithContainerID(options.ContainerID) - sops.WithSearchFilters(sfs) - return options.Client.SearchObject( - ctx, - sops, - client.WithSession(options.SessionToken), - client.WithBearer(options.BearerToken), - ) -} - -// Delete deletes NeoFS object. -func (oc *neofsObjectClient) Delete(ctx context.Context, options *DeleteOptions) error { - ops := new(client.DeleteObjectParams).WithAddress(options.ObjectAddress) - err := options.Client.DeleteObject( - ctx, - ops, - client.WithSession(options.SessionToken), - client.WithBearer(options.BearerToken), - ) - return err -} diff --git a/neofs/credentials.go b/neofs/credentials.go deleted file mode 100644 index c229468..0000000 --- a/neofs/credentials.go +++ /dev/null @@ -1,78 +0,0 @@ -package neofs - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "math/big" - - "github.com/nspcc-dev/neofs-api-go/pkg/owner" - crypto "github.com/nspcc-dev/neofs-crypto" -) - -type ( - // Credentials contains methods that needed to work with NeoFS. - Credentials interface { - Owner() *owner.ID - PublicKey() *ecdsa.PublicKey - PrivateKey() *ecdsa.PrivateKey - } - - credentials struct { - key *ecdsa.PrivateKey - ownerID *owner.ID - } -) - -// NewCredentials creates an instance of Credentials through string -// representation of secret. It allows passing WIF, path, hex-encoded and others. -func NewCredentials(secret string) (Credentials, error) { - key, err := crypto.LoadPrivateKey(secret) - if err != nil { - return nil, err - } - return setFromPrivateKey(key) -} - -// NewEphemeralCredentials creates new private key and Credentials based on that -// key. -func NewEphemeralCredentials() (Credentials, error) { - c := elliptic.P256() - priv, x, y, err := elliptic.GenerateKey(c, rand.Reader) - if err != nil { - return nil, err - } - key := &ecdsa.PrivateKey{ - PublicKey: ecdsa.PublicKey{ - Curve: c, - X: x, - Y: y, - }, - D: new(big.Int).SetBytes(priv), - } - return setFromPrivateKey(key) -} - -// PrivateKey returns ecdsa.PrivateKey. -func (c *credentials) PrivateKey() *ecdsa.PrivateKey { - return c.key -} - -// PublicKey returns ecdsa.PublicKey. -func (c *credentials) PublicKey() *ecdsa.PublicKey { - return &c.key.PublicKey -} - -// Owner returns owner.ID. -func (c *credentials) Owner() *owner.ID { - return c.ownerID -} - -func setFromPrivateKey(key *ecdsa.PrivateKey) (*credentials, error) { - wallet, err := owner.NEO3WalletFromPublicKey(&key.PublicKey) - if err != nil { - return nil, err - } - ownerID := owner.NewIDFromNeo3Wallet(wallet) - return &credentials{key: key, ownerID: ownerID}, nil -} diff --git a/uploader/upload.go b/uploader/upload.go index a9c874f..db5b7e8 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -5,15 +5,15 @@ import ( "encoding/json" "io" "strconv" - "sync" "time" + "github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/token" - "github.com/nspcc-dev/neofs-http-gw/neofs" "github.com/nspcc-dev/neofs-http-gw/tokens" + "github.com/nspcc-dev/neofs-sdk-go/pkg/neofs" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -23,12 +23,6 @@ const ( drainBufSize = 4096 ) -var putOptionsPool = sync.Pool{ - New: func() interface{} { - return new(neofs.PutOptions) - }, -} - // Uploader is an upload request handler. type Uploader struct { log *zap.Logger @@ -47,7 +41,10 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { var ( err error file MultipartFile - addr *object.Address + obj *object.ID + conn client.Client + tkn *token.SessionToken + addr = object.NewAddress() cid = container.NewID() scid, _ = c.UserValue("cid").(string) log = u.log.With(zap.String("cid", scid)) @@ -107,25 +104,31 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { attributes = append(attributes, timestamp) } oid, bt := u.fetchOwnerAndBearerToken(c) - putOpts := putOptionsPool.Get().(*neofs.PutOptions) - defer putOptionsPool.Put(putOpts) + // Try to put file into NeoFS or throw an error. - putOpts.Client, putOpts.SessionToken, err = u.plant.ConnectionArtifacts() + conn, tkn, err = u.plant.ConnectionArtifacts() if err != nil { log.Error("failed to get neofs connection artifacts", zap.Error(err)) c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) return } - putOpts.Attributes = attributes - putOpts.BearerToken = bt - putOpts.ContainerID = cid - putOpts.OwnerID = oid - putOpts.Reader = file - if addr, err = u.plant.Object().Put(c, putOpts); err != nil { + + rawObject := object.NewRaw() + rawObject.SetContainerID(cid) + rawObject.SetOwnerID(oid) + rawObject.SetAttributes(attributes...) + + ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(file) + + if obj, err = conn.PutObject(c, ops, client.WithSession(tkn), client.WithBearer(bt)); err != nil { log.Error("could not store file in neofs", zap.Error(err)) c.Error("could not store file in neofs", fasthttp.StatusBadRequest) return } + + addr.SetObjectID(obj) + addr.SetContainerID(cid) + // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { log.Error("could not prepare response", zap.Error(err)) @@ -151,8 +154,8 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { } func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*owner.ID, *token.BearerToken) { - if token, err := tokens.LoadBearerToken(ctx); err == nil && token != nil { - return token.Issuer(), token + if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil { + return tkn.Issuer(), tkn } return u.plant.OwnerID(), nil } From b402f5009e1b62a12e7fb111240a225c8e639e2a Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Fri, 28 May 2021 16:11:19 +0300 Subject: [PATCH 182/548] *: add short name for key parameter Signed-off-by: Angira Kekteeva --- settings.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/settings.go b/settings.go index 4642cb0..21de46e 100644 --- a/settings.go +++ b/settings.go @@ -104,8 +104,7 @@ func settings() *viper.Viper { help := flags.BoolP(cmdHelp, "h", false, "show help") version := flags.BoolP(cmdVersion, "v", false, "show version") - flags.String(cmdNeoFSKey, "", `path to private key file, hex string or wif (autogenerated key will be used if not specified)`) - + flags.StringP(cmdNeoFSKey, "k", "", `path to private key file, hex string or wif (autogenerated key will be used if not specified)`) flags.Bool(cmdVerbose, false, "debug gRPC connections") flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout") flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout") From e8b94553c32aef340203d53cba2bf56c0cc8f166 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 28 May 2021 23:24:04 +0300 Subject: [PATCH 183/548] *: update to use unneofsed sdk-go Signed-off-by: Roman Khimov --- app.go | 25 +++++++++++-------------- downloader/download.go | 16 ++++++++-------- go.mod | 3 ++- go.sum | 4 ++-- uploader/upload.go | 12 ++++++------ 5 files changed, 29 insertions(+), 31 deletions(-) diff --git a/app.go b/app.go index e086ed2..8e89484 100644 --- a/app.go +++ b/app.go @@ -2,14 +2,15 @@ package main import ( "context" + "crypto/ecdsa" "math" "strconv" "github.com/fasthttp/router" + crypto "github.com/nspcc-dev/neofs-crypto" "github.com/nspcc-dev/neofs-http-gw/downloader" "github.com/nspcc-dev/neofs-http-gw/uploader" "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" - "github.com/nspcc-dev/neofs-sdk-go/pkg/neofs" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/spf13/viper" "github.com/valyala/fasthttp" @@ -20,7 +21,7 @@ import ( type ( app struct { log *zap.Logger - plant neofs.ClientPlant + pool pool.Pool cfg *viper.Viper auxiliaryLog logger.Logger webServer *fasthttp.Server @@ -59,8 +60,8 @@ func WithConfig(c *viper.Viper) Option { func newApp(ctx context.Context, opt ...Option) App { var ( - creds neofs.Credentials - err error + key *ecdsa.PrivateKey + err error ) a := &app{ @@ -92,9 +93,9 @@ func newApp(ctx context.Context, opt ...Option) App { keystring := a.cfg.GetString(cmdNeoFSKey) if len(keystring) == 0 { a.log.Info("no key specified, creating one automatically for this run") - creds, err = neofs.NewEphemeralCredentials() + key, err = pool.NewEphemeralKey() } else { - creds, err = neofs.NewCredentials(keystring) + key, err = crypto.LoadPrivateKey(keystring) } if err != nil { a.log.Fatal("failed to get neofs credentials", zap.Error(err)) @@ -113,7 +114,7 @@ func newApp(ctx context.Context, opt ...Option) App { a.log.Info("add connection", zap.String("address", address), zap.Float64("weight", weight)) } opts := &pool.BuilderOptions{ - Key: creds.PrivateKey(), + Key: key, NodeConnectionTimeout: a.cfg.GetDuration(cfgConTimeout), NodeRequestTimeout: a.cfg.GetDuration(cfgReqTimeout), ClientRebalanceInterval: a.cfg.GetDuration(cfgRebalance), @@ -122,14 +123,10 @@ func newApp(ctx context.Context, opt ...Option) App { KeepaliveTimeout: a.cfg.GetDuration(cfgKeepaliveTimeout), KeepalivePermitWoStream: a.cfg.GetBool(cfgKeepalivePermitWithoutStream), } - pool, err := pb.Build(ctx, opts) + a.pool, err = pb.Build(ctx, opts) if err != nil { a.log.Fatal("failed to create connection pool", zap.Error(err)) } - a.plant, err = neofs.NewClientPlant(ctx, pool, creds) - if err != nil { - a.log.Fatal("failed to create neofs client plant") - } return a } @@ -145,8 +142,8 @@ func (a *app) Serve(ctx context.Context) { close(a.webDone) }() edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) - uploader := uploader.New(a.log, a.plant, edts) - downloader, err := downloader.New(ctx, a.log, a.plant) + uploader := uploader.New(a.log, a.pool, edts) + downloader, err := downloader.New(ctx, a.log, a.pool) if err != nil { a.log.Fatal("failed to create downloader", zap.Error(err)) } diff --git a/downloader/download.go b/downloader/download.go index 51baea2..ac33dcb 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -17,7 +17,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-http-gw/tokens" - "github.com/nspcc-dev/neofs-sdk-go/pkg/neofs" + "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" "google.golang.org/grpc/codes" @@ -164,14 +164,14 @@ func (o objectIDs) Slice() []string { // Downloader is a download request handler. type Downloader struct { - log *zap.Logger - plant neofs.ClientPlant + log *zap.Logger + pool pool.Pool } // New creates an instance of Downloader using specified options. -func New(ctx context.Context, log *zap.Logger, plant neofs.ClientPlant) (*Downloader, error) { +func New(ctx context.Context, log *zap.Logger, conns pool.Pool) (*Downloader, error) { var err error - d := &Downloader{log: log, plant: plant} + d := &Downloader{log: log, pool: conns} if err != nil { return nil, fmt.Errorf("failed to get neofs client's reusable artifacts: %w", err) } @@ -203,7 +203,7 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { return } - conn, tkn, err = d.plant.ConnectionArtifacts() + conn, tkn, err = d.pool.Connection() if err != nil { log.Error("failed to get neofs connection artifacts", zap.Error(err)) c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) @@ -231,7 +231,7 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { return } - conn, tkn, err = d.plant.ConnectionArtifacts() + conn, tkn, err = d.pool.Connection() if err != nil { log.Error("failed to get neofs connection artifacts", zap.Error(err)) c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) @@ -261,7 +261,7 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { address.SetContainerID(cid) address.SetObjectID(ids[0]) - conn, tkn, err = d.plant.ConnectionArtifacts() + conn, tkn, err = d.pool.Connection() if err != nil { log.Error("failed to get neofs connection artifacts", zap.Error(err)) c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) diff --git a/go.mod b/go.mod index 059c184..3008daf 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,8 @@ require ( github.com/fasthttp/router v1.3.5 github.com/mr-tron/base58 v1.1.3 // indirect github.com/nspcc-dev/neofs-api-go v1.26.1 - github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210527182636-cbfc17a1a9a2 + github.com/nspcc-dev/neofs-crypto v0.3.0 + github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210531182150-2f63343bda48 github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 6459a87..22eee4b 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9K github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210527182636-cbfc17a1a9a2 h1:z8xtKILKi+Dolk3VAyCaFPMroFnT+x8qTqMT/zBRqIc= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210527182636-cbfc17a1a9a2/go.mod h1:QZE7VaNQRyNFS+3gsrNEQEiLe+d6AR6EteX1M9geh6A= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210531182150-2f63343bda48 h1:IcgVSj33HSH621/K0Jk9JTFJ/ooIs2d4hd+kww5d/O8= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210531182150-2f63343bda48/go.mod h1:QZE7VaNQRyNFS+3gsrNEQEiLe+d6AR6EteX1M9geh6A= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/uploader/upload.go b/uploader/upload.go index db5b7e8..7a1ff29 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -13,7 +13,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-http-gw/tokens" - "github.com/nspcc-dev/neofs-sdk-go/pkg/neofs" + "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -26,14 +26,14 @@ const ( // Uploader is an upload request handler. type Uploader struct { log *zap.Logger - plant neofs.ClientPlant + pool pool.Pool enableDefaultTimestamp bool } // New creates a new Uploader using specified logger, connection pool and // other options. -func New(log *zap.Logger, plant neofs.ClientPlant, enableDefaultTimestamp bool) *Uploader { - return &Uploader{log, plant, enableDefaultTimestamp} +func New(log *zap.Logger, conns pool.Pool, enableDefaultTimestamp bool) *Uploader { + return &Uploader{log, conns, enableDefaultTimestamp} } // Upload handles multipart upload request. @@ -106,7 +106,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { oid, bt := u.fetchOwnerAndBearerToken(c) // Try to put file into NeoFS or throw an error. - conn, tkn, err = u.plant.ConnectionArtifacts() + conn, tkn, err = u.pool.Connection() if err != nil { log.Error("failed to get neofs connection artifacts", zap.Error(err)) c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) @@ -157,7 +157,7 @@ func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*owner.ID, *to if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil { return tkn.Issuer(), tkn } - return u.plant.OwnerID(), nil + return u.pool.OwnerID(), nil } type putResponse struct { From b93fa7048a7bdc30bd5204cf42900ee23922b2c7 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 3 Jun 2021 15:18:27 +0300 Subject: [PATCH 184/548] [#53] Fixed bearer cookie key in README.md Signed-off-by: Denis Kirillov --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ccbba33..3534306 100644 --- a/README.md +++ b/README.md @@ -355,7 +355,7 @@ NeoFS Bearer Token basically is a container owner-signed ACL data (refer to NeoF documentation for more details). There are two options to pass them to gateway: * "Authorization" header with "Bearer" type and base64-encoded token in credentials field - * "__context_bearer_token_key" cookie with base64-encoded token contents + * "Bearer" cookie with base64-encoded token contents For example you have a mobile application frontend with a backend part storing data in NeoFS. When user authorizes in mobile app, the backend issues a NeoFS From 2510d1054a54f06363d2729bfbcd4f196dc9b881 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Fri, 4 Jun 2021 10:38:01 +0300 Subject: [PATCH 185/548] readme: update params of acl 'create a container' Signed-off-by: Angira Kekteeva --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3534306..df15ae5 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ $ neofs-cli -r $NEOFS_NODE -k $KEY container create --policy $POLICY --basic-acl ``` where `$KEY` can be a path to private key file (as raw bytes), a hex string or (unencrypted) WIF string, -`$ACL` -- hex encoded basic ACL value or keywords 'private, 'public', 'readonly' and +`$ACL` -- hex encoded basic ACL value or keywords 'private, 'public-read', 'public-read-write' and `$POLICY` -- QL-encoded or JSON-encoded placement policy or path to file with it For example: From 124a30eec77790ea8e7bdc936301d8d43d84da07 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Fri, 4 Jun 2021 15:55:02 +0300 Subject: [PATCH 186/548] [#57] go.mod: Update api-go and sdk-go versions Signed-off-by: Angira Kekteeva --- go.mod | 4 ++-- go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 3008daf..12535bb 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.16 require ( github.com/fasthttp/router v1.3.5 github.com/mr-tron/base58 v1.1.3 // indirect - github.com/nspcc-dev/neofs-api-go v1.26.1 + github.com/nspcc-dev/neofs-api-go v1.27.0 github.com/nspcc-dev/neofs-crypto v0.3.0 - github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210531182150-2f63343bda48 + github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210604112451-f16d38c7b92a github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 22eee4b..9d80a35 100644 --- a/go.sum +++ b/go.sum @@ -310,17 +310,17 @@ github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neo-go v0.95.0 h1:bttArYkIuhBJWSZsZ1xVW8MJsj5SvZwAhqVN3HZPNbo= -github.com/nspcc-dev/neo-go v0.95.0/go.mod h1:bW07ge1WFXsBgqrcPpLUr6OcyQxHqM26MZNesWMdH0c= +github.com/nspcc-dev/neo-go v0.95.1 h1:5fgLFOul1Ax/maFkgLkD5rDUwY/nB/xX/Jpcd8hLHaI= +github.com/nspcc-dev/neo-go v0.95.1/go.mod h1:bW07ge1WFXsBgqrcPpLUr6OcyQxHqM26MZNesWMdH0c= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= -github.com/nspcc-dev/neofs-api-go v1.26.1 h1:GMIuEB6Hv9IXP9SJd/1f8Df6gRriPkSplpmpJXgQ/1I= -github.com/nspcc-dev/neofs-api-go v1.26.1/go.mod h1:SHuH1Ba3U/h3j+8HHbb3Cns1LfMlEb88guWog9Qi68Y= +github.com/nspcc-dev/neofs-api-go v1.27.0 h1:SiqD1wb50l/ahCNV8/D9R3ua/sFS8oRCJ5jV+ux6AzE= +github.com/nspcc-dev/neofs-api-go v1.27.0/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210531182150-2f63343bda48 h1:IcgVSj33HSH621/K0Jk9JTFJ/ooIs2d4hd+kww5d/O8= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210531182150-2f63343bda48/go.mod h1:QZE7VaNQRyNFS+3gsrNEQEiLe+d6AR6EteX1M9geh6A= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210604112451-f16d38c7b92a h1:bVvyR+Y+UmElTFKY0ifjtvWteYSm93jihKV1rh4wW5s= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210604112451-f16d38c7b92a/go.mod h1:1djNrOkpTTbNUlJM/MvTmohJUaWKUMy9JHSCCA8rJEc= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= From 2f8ab7cb8dfac2526a985cfac222489a32022dd4 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Fri, 4 Jun 2021 15:55:56 +0300 Subject: [PATCH 187/548] [#57] downloader, uploader: Replace deprecated Signed-off-by: Angira Kekteeva --- downloader/download.go | 12 ++++++------ uploader/upload.go | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index ac33dcb..97e8658 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -13,9 +13,9 @@ import ( "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" - "github.com/nspcc-dev/neofs-api-go/pkg/container" + cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-api-go/pkg/session" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/valyala/fasthttp" @@ -73,7 +73,7 @@ func isValidValue(s string) bool { } func (r *request) receiveFile(clnt client.Client, - sessionToken *token.SessionToken, + sessionToken *session.Token, objectAddress *object.Address) { var ( err error @@ -195,7 +195,7 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { val = strings.Join([]string{cid, oid}, "/") log = d.log.With(zap.String("cid", cid), zap.String("oid", oid)) conn client.Client - tkn *token.SessionToken + tkn *session.Token ) if err = address.Parse(val); err != nil { log.Error("wrong object address", zap.Error(err)) @@ -222,9 +222,9 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ids []*object.ID conn client.Client - tkn *token.SessionToken + tkn *session.Token ) - cid := container.NewID() + cid := cid.New() if err = cid.Parse(scid); err != nil { log.Error("wrong container id", zap.Error(err)) c.Error("wrong container id", fasthttp.StatusBadRequest) diff --git a/uploader/upload.go b/uploader/upload.go index 7a1ff29..60d7003 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -8,9 +8,10 @@ import ( "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" - "github.com/nspcc-dev/neofs-api-go/pkg/container" + cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-api-go/pkg/session" "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" @@ -43,9 +44,9 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { file MultipartFile obj *object.ID conn client.Client - tkn *token.SessionToken + tkn *session.Token addr = object.NewAddress() - cid = container.NewID() + cid = cid.New() scid, _ = c.UserValue("cid").(string) log = u.log.With(zap.String("cid", scid)) bodyStream = c.RequestBodyStream() From 742eb7d0d710939199d4351ce9a04ed562565a06 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 3 Jun 2021 17:11:54 +0300 Subject: [PATCH 188/548] [#55] Bearer token example Example of using console utils to generate token Signed-off-by: Denis Kirillov --- README.md | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/README.md b/README.md index ccbba33..7688954 100644 --- a/README.md +++ b/README.md @@ -364,6 +364,79 @@ some data and upload it via any available NeoFS HTTP Protocol Gateway by adding the corresponding header to the upload request. Accessing the ACL protected data works the same way. +##### Example +In order to generate bearer token, you need to know container owner key and +address of sender who will be do request to NeoFS (in our case it's gateway wallet address). + +Suppose we have: +* **KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr** (container owner key) +* **NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3** (token owner address) +* **BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K** (container id) + +Firstly we need to encode container id and sender address to base64 (now it's base58). +So use **base58** and **base64** utils. + +1. Encoding container id: +``` +$ echo 'BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K' | base58 --decode | base64 +# output: mRnZWzewzxjzIPa7Fqlfqdl3TM1KpJ0YnsXsEhafJJg= +``` + +2. Encoding token owner id: +``` +$ echo 'NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3' | base58 --decode | base64 +# output: NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg== +``` + +Now we can form Bearer token (10000 is liftetime expiration in epoch) and save it to **bearer.json**: +``` +{ + "body": { + "eaclTable": { + "version": { + "major": 0, + "minor": 0 + }, + "containerID": { + "value": "mRnZWzewzxjzIPa7Fqlfqdl3TM1KpJ0YnsXsEhafJJg=" + }, + "records": [] + }, + "ownerID": { + "value": "NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg==" + }, + "lifetime": { + "exp": "10000", + "nbf": "0", + "iat": "0" + } + }, + "signature": null +} +``` + +Then sign it with container owner key: +``` +$ neofs-cli util sign bearer-token --from bearer.json --to signed.json -k KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr +``` +Encoding to base64 to use via header: +``` +$ base64 -w 0 signed.json +# output: Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw== +``` + +After that Bearer token can be used: + +``` +$ curl -F 'file=@cat.jpeg;filename=cat.jpeg' -H "Authorization: Bearer Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw==" \ + http://localhost:8082/upload/BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K +# output: +# { +# "object_id": "DhfES9nVrFksxGDD2jQLunGADfrXExxNwqXbDafyBn9X", +# "container_id": "BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K" +# } +``` + ### Metrics and Pprof If enabled, Prometheus metrics are available at `/metrics/` path and Pprof at From 63e9048de6c6005a7628f142f84369c9be60767b Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 7 Jun 2021 15:37:03 +0300 Subject: [PATCH 189/548] [#XX] Added Bearer token requirements Signed-off-by: Denis Kirillov --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index 8b09de0..5ff0445 100644 --- a/README.md +++ b/README.md @@ -437,6 +437,47 @@ $ curl -F 'file=@cat.jpeg;filename=cat.jpeg' -H "Authorization: Bearer Ck4KKgoEC # } ``` +##### Note +For the token to work correctly, you need to create a container with a basic ACL that: +1. Allow PUT operation to others +2. Doesn't set "final" bit + +For example: +``` +$ neofs-cli --key KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr --basic-acl 0x0FFFCFFF -r 192.168.130.72:8080 container create --policy "REP 3" --await +``` + +To deny access to the container without a token, set the eACL rules: +``` +$ neofs-cli --key KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr -r 192.168.130.72:8080 container set-eacl --table eacl.json --await --cid BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K +``` + +File **eacl.json**: +``` +{ + "version": { + "major": 0, + "minor": 0 + }, + "containerID": { + "value": "mRnZWzewzxjzIPa7Fqlfqdl3TM1KpJ0YnsXsEhafJJg=" + }, + "records": [ + { + "operation": "PUT", + "action": "DENY", + "filters": [], + "targets": [ + { + "role": "OTHERS", + "keys": [] + } + ] + } + ] +} +``` + ### Metrics and Pprof If enabled, Prometheus metrics are available at `/metrics/` path and Pprof at From 7734be609cac88c49f57c68d98d8d5e69da29ed9 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 9 Jun 2021 19:26:27 +0300 Subject: [PATCH 190/548] [#60] Dropped grpc keepalive options Signed-off-by: Denis Kirillov --- README.md | 9 ++++++--- app.go | 3 --- go.mod | 2 +- go.sum | 4 ++-- settings.go | 14 -------------- 5 files changed, 9 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 5ff0445..a757896 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,11 @@ backend node (and otherwise default settings): $ neofs-http-gw -p 192.168.130.72:8080 $ HTTP_GW_PEERS_0_ADDRESS=192.168.130.72:8080 neofs-http-gw ``` +It's also possible to specify uri scheme (grpc or grpcs) when using `-p`: +``` +$ neofs-http-gw -p grpc://192.168.130.72:8080 +$ HTTP_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 neofs-http-gw +``` ## Configuration @@ -145,9 +150,7 @@ default. To enable them use `--pprof` and `--metrics` flags or You can tune gRPC interface parameters with `--connect_timeout` (for connection to node) and `--request_timeout` (for request processing over -established connection) options as well as `HTTP_GW_KEEPALIVE_TIME` -(peer pinging interval), `HTTP_GW_KEEPALIVE_TIMEOUT` (peer pinging timeout) -and `HTTP_GW_KEEPALIVE_PERMIT_WITHOUT_STREAM` environment variables. +established connection) options. gRPC-level checks allow gateway to detect dead peers, but it declares them unhealthy at pool level once per `--rebalance_timer` interval, so check for it diff --git a/app.go b/app.go index 8e89484..86da641 100644 --- a/app.go +++ b/app.go @@ -119,9 +119,6 @@ func newApp(ctx context.Context, opt ...Option) App { NodeRequestTimeout: a.cfg.GetDuration(cfgReqTimeout), ClientRebalanceInterval: a.cfg.GetDuration(cfgRebalance), SessionExpirationEpoch: math.MaxUint64, - KeepaliveTime: a.cfg.GetDuration(cfgKeepaliveTime), - KeepaliveTimeout: a.cfg.GetDuration(cfgKeepaliveTimeout), - KeepalivePermitWoStream: a.cfg.GetBool(cfgKeepalivePermitWithoutStream), } a.pool, err = pb.Build(ctx, opts) if err != nil { diff --git a/go.mod b/go.mod index 12535bb..2aaa8e3 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/mr-tron/base58 v1.1.3 // indirect github.com/nspcc-dev/neofs-api-go v1.27.0 github.com/nspcc-dev/neofs-crypto v0.3.0 - github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210604112451-f16d38c7b92a + github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210609143631-0d3c078a0d9b github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 9d80a35..90c40de 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9K github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210604112451-f16d38c7b92a h1:bVvyR+Y+UmElTFKY0ifjtvWteYSm93jihKV1rh4wW5s= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210604112451-f16d38c7b92a/go.mod h1:1djNrOkpTTbNUlJM/MvTmohJUaWKUMy9JHSCCA8rJEc= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210609143631-0d3c078a0d9b h1:2alc6tGPHScEATOxlrYuHCTl+DbhVaqigT5Bo1QXY90= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210609143631-0d3c078a0d9b/go.mod h1:1djNrOkpTTbNUlJM/MvTmohJUaWKUMy9JHSCCA8rJEc= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/settings.go b/settings.go index 21de46e..bf26d2c 100644 --- a/settings.go +++ b/settings.go @@ -23,18 +23,10 @@ const ( defaultRequestTimeout = 15 * time.Second defaultConnectTimeout = 30 * time.Second - defaultKeepaliveTime = 10 * time.Second - defaultKeepaliveTimeout = 10 * time.Second - cfgListenAddress = "listen_address" cfgTLSCertificate = "tls_certificate" cfgTLSKey = "tls_key" - // KeepAlive. - cfgKeepaliveTime = "keepalive.time" - cfgKeepaliveTimeout = "keepalive.timeout" - cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream" - // Web. cfgWebReadBufferSize = "web.read_buffer_size" cfgWebWriteBufferSize = "web.write_buffer_size" @@ -130,12 +122,6 @@ func settings() *viper.Viper { v.SetDefault(cfgLoggerSamplingInitial, 1000) v.SetDefault(cfgLoggerSamplingThereafter, 1000) - // keepalive: - // If set below 10s, a minimum value of 10s will be used instead. - v.SetDefault(cfgKeepaliveTime, defaultKeepaliveTime) - v.SetDefault(cfgKeepaliveTimeout, defaultKeepaliveTimeout) - v.SetDefault(cfgKeepalivePermitWithoutStream, true) - // web-server: v.SetDefault(cfgWebReadBufferSize, 4096) v.SetDefault(cfgWebWriteBufferSize, 4096) From e177e1c9ae1e2683fa41fc548708f669f4e8fcca Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 11 Jun 2021 11:54:00 +0300 Subject: [PATCH 191/548] *: use client.Object where appropriate It's enough to do the job, we don't really need full client.Client interface here. Signed-off-by: Roman Khimov --- downloader/download.go | 6 +++--- uploader/upload.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 97e8658..ab0b762 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -72,7 +72,7 @@ func isValidValue(s string) bool { return true } -func (r *request) receiveFile(clnt client.Client, +func (r *request) receiveFile(clnt client.Object, sessionToken *session.Token, objectAddress *object.Address) { var ( @@ -194,7 +194,7 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { oid, _ = c.UserValue("oid").(string) val = strings.Join([]string{cid, oid}, "/") log = d.log.With(zap.String("cid", cid), zap.String("oid", oid)) - conn client.Client + conn client.Object tkn *session.Token ) if err = address.Parse(val); err != nil { @@ -221,7 +221,7 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { val, _ = c.UserValue("attr_val").(string) log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ids []*object.ID - conn client.Client + conn client.Object tkn *session.Token ) cid := cid.New() diff --git a/uploader/upload.go b/uploader/upload.go index 60d7003..e8f9735 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -43,7 +43,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { err error file MultipartFile obj *object.ID - conn client.Client + conn client.Object tkn *session.Token addr = object.NewAddress() cid = cid.New() From fffb0b2f12d09974f26773b523191416173d13e8 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 15 Jun 2021 11:28:30 +0300 Subject: [PATCH 192/548] [#63] Using client.Object from sdk Signed-off-by: Denis Kirillov --- downloader/download.go | 35 ++++------------------------------- go.mod | 2 +- go.sum | 2 ++ uploader/upload.go | 13 +------------ 4 files changed, 8 insertions(+), 44 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index ab0b762..75c5f8a 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -15,7 +15,6 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/client" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neofs-api-go/pkg/session" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/valyala/fasthttp" @@ -72,9 +71,7 @@ func isValidValue(s string) bool { return true } -func (r *request) receiveFile(clnt client.Object, - sessionToken *session.Token, - objectAddress *object.Address) { +func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address) { var ( err error dis = "inline" @@ -95,7 +92,6 @@ func (r *request) receiveFile(clnt client.Object, obj, err = clnt.GetObject( r.RequestCtx, options, - client.WithSession(sessionToken), ) if err != nil { r.log.Error( @@ -194,8 +190,6 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { oid, _ = c.UserValue("oid").(string) val = strings.Join([]string{cid, oid}, "/") log = d.log.With(zap.String("cid", cid), zap.String("oid", oid)) - conn client.Object - tkn *session.Token ) if err = address.Parse(val); err != nil { log.Error("wrong object address", zap.Error(err)) @@ -203,13 +197,7 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { return } - conn, tkn, err = d.pool.Connection() - if err != nil { - log.Error("failed to get neofs connection artifacts", zap.Error(err)) - c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) - return - } - d.newRequest(c, log).receiveFile(conn, tkn, address) + d.newRequest(c, log).receiveFile(d.pool, address) } // DownloadByAttribute handles attribute-based download requests. @@ -221,8 +209,6 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { val, _ = c.UserValue("attr_val").(string) log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ids []*object.ID - conn client.Object - tkn *session.Token ) cid := cid.New() if err = cid.Parse(scid); err != nil { @@ -231,19 +217,12 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { return } - conn, tkn, err = d.pool.Connection() - if err != nil { - log.Error("failed to get neofs connection artifacts", zap.Error(err)) - c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) - return - } - options := object.NewSearchFilters() options.AddRootFilter() options.AddFilter(key, val, object.MatchStringEqual) sops := new(client.SearchObjectParams).WithContainerID(cid).WithSearchFilters(options) - if ids, err = conn.SearchObject(c, sops, client.WithSession(tkn)); err != nil { + if ids, err = d.pool.SearchObject(c, sops); err != nil { log.Error("something went wrong", zap.Error(err)) c.Error("something went wrong", fasthttp.StatusBadRequest) return @@ -261,11 +240,5 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { address.SetContainerID(cid) address.SetObjectID(ids[0]) - conn, tkn, err = d.pool.Connection() - if err != nil { - log.Error("failed to get neofs connection artifacts", zap.Error(err)) - c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) - return - } - d.newRequest(c, log).receiveFile(conn, tkn, address) + d.newRequest(c, log).receiveFile(d.pool, address) } diff --git a/go.mod b/go.mod index 2aaa8e3..2ad26ef 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/mr-tron/base58 v1.1.3 // indirect github.com/nspcc-dev/neofs-api-go v1.27.0 github.com/nspcc-dev/neofs-crypto v0.3.0 - github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210609143631-0d3c078a0d9b + github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 90c40de..03868e3 100644 --- a/go.sum +++ b/go.sum @@ -321,6 +321,8 @@ github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnB github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210609143631-0d3c078a0d9b h1:2alc6tGPHScEATOxlrYuHCTl+DbhVaqigT5Bo1QXY90= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210609143631-0d3c078a0d9b/go.mod h1:1djNrOkpTTbNUlJM/MvTmohJUaWKUMy9JHSCCA8rJEc= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b h1:l99sCKR/mt+iFb4p1836qtoXUQEGYlEzqWAeAliEaE8= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b/go.mod h1:1djNrOkpTTbNUlJM/MvTmohJUaWKUMy9JHSCCA8rJEc= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/uploader/upload.go b/uploader/upload.go index e8f9735..f43dfb2 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -11,7 +11,6 @@ import ( cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" - "github.com/nspcc-dev/neofs-api-go/pkg/session" "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" @@ -43,8 +42,6 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { err error file MultipartFile obj *object.ID - conn client.Object - tkn *session.Token addr = object.NewAddress() cid = cid.New() scid, _ = c.UserValue("cid").(string) @@ -106,14 +103,6 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { } oid, bt := u.fetchOwnerAndBearerToken(c) - // Try to put file into NeoFS or throw an error. - conn, tkn, err = u.pool.Connection() - if err != nil { - log.Error("failed to get neofs connection artifacts", zap.Error(err)) - c.Error("failed to get neofs connection artifacts", fasthttp.StatusInternalServerError) - return - } - rawObject := object.NewRaw() rawObject.SetContainerID(cid) rawObject.SetOwnerID(oid) @@ -121,7 +110,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(file) - if obj, err = conn.PutObject(c, ops, client.WithSession(tkn), client.WithBearer(bt)); err != nil { + if obj, err = u.pool.PutObject(c, ops, client.WithBearer(bt)); err != nil { log.Error("could not store file in neofs", zap.Error(err)) c.Error("could not store file in neofs", fasthttp.StatusBadRequest) return From 9a49e2c85e0b8786f5f34ad3ac9fed2409c327c5 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 21 Jun 2021 16:56:51 +0300 Subject: [PATCH 193/548] [#65] Supported object Content-Type Signed-off-by: Denis Kirillov --- downloader/download.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/downloader/download.go b/downloader/download.go index 75c5f8a..b8ed12a 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -141,6 +141,8 @@ func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address) } r.Response.Header.Set("Last-Modified", time.Unix(value, 0).Format(time.RFC1123)) + case object.AttributeContentType: + writer.contentType = val } } r.Response.Header.Set("x-object-id", obj.ID().String()) From 42655cf476153a4cf519698c91583086c5f4d03e Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 22 Jun 2021 13:47:38 +0300 Subject: [PATCH 194/548] CHANGELOG: release 0.15.2 --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a3c611..c699f2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ This document outlines major changes between releases. +## 0.15.2 (22 Jun 2021) + +New features: + * Content-Type returned for object GET requests can now be taken from + attributes (overriding autodetection, #65) + +Behavior changes: + * grpc keepalive options can no longer be changed (#60) + +Improvements: + * code refactoring (more reuse between different gateways, moved some code to + sdk-go, #47, #46, #51, #62, #63) + * documentation updates and fixes (#53, #49, #55, #59) + * updated api-go dependency (#57) + +Bugs fixed: + * `-k` option wasn't accepted for key although it was documented (#50) + ## 0.15.1 (24 May 2021) This important release makes HTTP gateway compatible with NeoFS node version From b1c72b83a90648624baa57aff7a5ef828abadc89 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 22 Jun 2021 13:55:55 +0300 Subject: [PATCH 195/548] go.sum: go mod tidy Signed-off-by: Roman Khimov --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 03868e3..cd20cd8 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,6 @@ github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9K github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210609143631-0d3c078a0d9b h1:2alc6tGPHScEATOxlrYuHCTl+DbhVaqigT5Bo1QXY90= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210609143631-0d3c078a0d9b/go.mod h1:1djNrOkpTTbNUlJM/MvTmohJUaWKUMy9JHSCCA8rJEc= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b h1:l99sCKR/mt+iFb4p1836qtoXUQEGYlEzqWAeAliEaE8= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b/go.mod h1:1djNrOkpTTbNUlJM/MvTmohJUaWKUMy9JHSCCA8rJEc= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= From 2d41fbc95e3626dca3b7bf9bad2f60cf64609265 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 22 Jun 2021 14:53:59 +0300 Subject: [PATCH 196/548] [#68] Support NEP-6 wallets Signed-off-by: Denis Kirillov --- README.md | 15 +++++------- app.go | 66 ++++++++++++++++++++++++++++++++++++++++++++++------- go.mod | 4 ++-- go.sum | 34 +++++++++++++++++++++++---- settings.go | 19 +++++++++------ 5 files changed, 108 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index a757896..17a5735 100644 --- a/README.md +++ b/README.md @@ -88,18 +88,15 @@ This command will make gateway use 192.168.130.72 for 90% of requests and 192.168.130.71 for remaining 10%. ### Keys - -By default gateway autogenerates key pair it will use for NeoFS requests. If -for some reason you need to have static keys you can pass them via `--key` -(or `-k`) parameter. The key can be a path to private key file (as raw bytes), -a hex string or (unencrypted) WIF string: +You can provide wallet via `--wallet` or `-w` flag also you can specify account address using `--address` +(if no address provided default one will be used). If wallet is used you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt wallet. +If no wallet provided gateway autogenerates key pair it will use for NeoFS requests. ``` -$ neofs-http-gw -p $NEOFS_NODE -k $KEY +$ neofs-http-gw -p $NEOFS_NODE -w $WALLET_PATH --address $ACCOUNT_ADDRESS ``` -For example: - +Example: ``` -$ neofs-http-gw -p 192.168.130.72:8080 -k KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr +$ neofs-http-gw -p 192.168.130.72:8080 -w wallet.json --address NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP ``` ### Binding and TLS diff --git a/app.go b/app.go index 86da641..0fbc312 100644 --- a/app.go +++ b/app.go @@ -3,11 +3,15 @@ package main import ( "context" "crypto/ecdsa" + "fmt" "math" "strconv" "github.com/fasthttp/router" - crypto "github.com/nspcc-dev/neofs-crypto" + "github.com/nspcc-dev/neo-go/cli/flags" + "github.com/nspcc-dev/neo-go/cli/input" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/nspcc-dev/neofs-http-gw/downloader" "github.com/nspcc-dev/neofs-http-gw/uploader" "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" @@ -90,13 +94,7 @@ func newApp(ctx context.Context, opt ...Option) App { a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - keystring := a.cfg.GetString(cmdNeoFSKey) - if len(keystring) == 0 { - a.log.Info("no key specified, creating one automatically for this run") - key, err = pool.NewEphemeralKey() - } else { - key, err = crypto.LoadPrivateKey(keystring) - } + key, err = getNeoFSKey(a) if err != nil { a.log.Fatal("failed to get neofs credentials", zap.Error(err)) } @@ -127,6 +125,58 @@ func newApp(ctx context.Context, opt ...Option) App { return a } +func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) { + walletPath := a.cfg.GetString(cmdWallet) + if len(walletPath) == 0 { + a.log.Info("no wallet path specified, creating ephemeral key automatically for this run") + return pool.NewEphemeralKey() + } + w, err := wallet.NewWalletFromFile(walletPath) + if err != nil { + return nil, err + } + + var password *string + if a.cfg.IsSet(cfgWalletPassphrase) { + pwd := a.cfg.GetString(cfgWalletPassphrase) + password = &pwd + } + return getKeyFromWallet(w, a.cfg.GetString(cmdAddress), password) +} + +func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*ecdsa.PrivateKey, error) { + var addr util.Uint160 + var err error + + if addrStr == "" { + addr = w.GetChangeAddress() + } else { + addr, err = flags.ParseAddress(addrStr) + if err != nil { + return nil, fmt.Errorf("invalid address") + } + } + + acc := w.GetAccount(addr) + if acc == nil { + return nil, fmt.Errorf("couldn't find wallet account for %s", addrStr) + } + + if password == nil { + pwd, err := input.ReadPassword("Enter password > ") + if err != nil { + return nil, fmt.Errorf("couldn't read password") + } + password = &pwd + } + + if err := acc.Decrypt(*password, w.Scrypt); err != nil { + return nil, fmt.Errorf("couldn't decrypt account: %w", err) + } + + return &acc.PrivateKey().PrivateKey, nil +} + func (a *app) Wait() { a.log.Info("starting application") <-a.webDone // wait for web-server to be stopped diff --git a/go.mod b/go.mod index 2ad26ef..78d6162 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.16 require ( github.com/fasthttp/router v1.3.5 github.com/mr-tron/base58 v1.1.3 // indirect - github.com/nspcc-dev/neofs-api-go v1.27.0 - github.com/nspcc-dev/neofs-crypto v0.3.0 + github.com/nspcc-dev/neo-go v0.95.3 + github.com/nspcc-dev/neofs-api-go v1.27.1 github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 diff --git a/go.sum b/go.sum index 03868e3..c053121 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,10 @@ github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1 github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -36,7 +38,9 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 h1:45bxf7AZMwWcqkLzDAQugVEwedisr5nRJ1r+7LYnv0U= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= @@ -92,19 +96,25 @@ github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI= github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU= github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -134,6 +144,7 @@ github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgO github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-redis/redis v6.10.2+incompatible h1:SLbqrO/Ik1nhXA5/cbEs1P5MUBo1Qq4ihlNfGnnipPw= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -164,7 +175,9 @@ github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -223,6 +236,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -310,17 +324,17 @@ github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neo-go v0.95.1 h1:5fgLFOul1Ax/maFkgLkD5rDUwY/nB/xX/Jpcd8hLHaI= github.com/nspcc-dev/neo-go v0.95.1/go.mod h1:bW07ge1WFXsBgqrcPpLUr6OcyQxHqM26MZNesWMdH0c= +github.com/nspcc-dev/neo-go v0.95.3 h1:RxBKcmmatbSM2cETGhv3ritmrkU0gUnWItNZvtrBtI0= +github.com/nspcc-dev/neo-go v0.95.3/go.mod h1:t15xRFDVhz5o/pstptdoW9N9JJBNn1hZ6APMNiC6MrY= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= -github.com/nspcc-dev/neofs-api-go v1.27.0 h1:SiqD1wb50l/ahCNV8/D9R3ua/sFS8oRCJ5jV+ux6AzE= github.com/nspcc-dev/neofs-api-go v1.27.0/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= +github.com/nspcc-dev/neofs-api-go v1.27.1 h1:ONdKOnm0/hK6m38VTUliCHY6RTxg+IpAzY4G+BeOZG4= +github.com/nspcc-dev/neofs-api-go v1.27.1/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210609143631-0d3c078a0d9b h1:2alc6tGPHScEATOxlrYuHCTl+DbhVaqigT5Bo1QXY90= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210609143631-0d3c078a0d9b/go.mod h1:1djNrOkpTTbNUlJM/MvTmohJUaWKUMy9JHSCCA8rJEc= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b h1:l99sCKR/mt+iFb4p1836qtoXUQEGYlEzqWAeAliEaE8= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b/go.mod h1:1djNrOkpTTbNUlJM/MvTmohJUaWKUMy9JHSCCA8rJEc= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= @@ -332,9 +346,11 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= @@ -399,13 +415,16 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/savsgio/gotils v0.0.0-20210105085219-0567298fdcac h1:5pr1F6tcjeIKPKlrQZW6hzB0WxZge7SkkYVGG/e5pLY= github.com/savsgio/gotils v0.0.0-20210105085219-0567298fdcac/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -448,11 +467,13 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73 h1:I2drr5K0tykBofr74ZEGliE/Hf6fNkEbcPyFvsy7wZk= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -461,9 +482,11 @@ github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= +github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7 h1:Y17pEjKgx2X0A69WQPGa8hx/Myzu+4NdUxlkZpbAYio= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -590,6 +613,7 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWB golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -687,11 +711,13 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= diff --git a/settings.go b/settings.go index bf26d2c..983ac84 100644 --- a/settings.go +++ b/settings.go @@ -49,6 +49,9 @@ const ( cfgLoggerSamplingInitial = "logger.sampling.initial" cfgLoggerSamplingThereafter = "logger.sampling.thereafter" + // Wallet. + cfgWalletPassphrase = "wallet.passphrase" + // Uploader Header. cfgUploaderHeaderEnableDefaultTimestamp = "upload_header.use_default_timestamp" @@ -60,12 +63,13 @@ const ( cfgApplicationVersion = "app.version" // Command line args. - cmdHelp = "help" - cmdVersion = "version" - cmdVerbose = "verbose" - cmdPprof = "pprof" - cmdMetrics = "metrics" - cmdNeoFSKey = "key" + cmdHelp = "help" + cmdVersion = "version" + cmdVerbose = "verbose" + cmdPprof = "pprof" + cmdMetrics = "metrics" + cmdWallet = "wallet" + cmdAddress = "address" ) var ignore = map[string]struct{}{ @@ -96,7 +100,8 @@ func settings() *viper.Viper { help := flags.BoolP(cmdHelp, "h", false, "show help") version := flags.BoolP(cmdVersion, "v", false, "show version") - flags.StringP(cmdNeoFSKey, "k", "", `path to private key file, hex string or wif (autogenerated key will be used if not specified)`) + flags.StringP(cmdWallet, "w", "", `path to the wallet`) + flags.String(cmdAddress, "", `address of wallet account`) flags.Bool(cmdVerbose, false, "debug gRPC connections") flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout") flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout") From 5eed0fd22d9f951cf84ee99ac47dab128be8daa5 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 23 Jun 2021 13:51:53 +0300 Subject: [PATCH 197/548] [#67] Added streaming to download Signed-off-by: Denis Kirillov --- downloader/download.go | 85 ++++++++++++++++++++++++++++++++------- downloader/reader_test.go | 43 ++++++++++++++++++++ go.mod | 2 +- go.sum | 3 +- 4 files changed, 117 insertions(+), 16 deletions(-) create mode 100644 downloader/reader_test.go diff --git a/downloader/download.go b/downloader/download.go index b8ed12a..079244a 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -9,7 +9,6 @@ import ( "path" "strconv" "strings" - "sync" "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" @@ -25,9 +24,11 @@ import ( type ( detector struct { - io.Writer - sync.Once + io.Reader + err error contentType string + done chan struct{} + data []byte } request struct { @@ -36,17 +37,57 @@ type ( } objectIDs []*object.ID + + errReader struct { + data []byte + err error + offset int + } ) -func newDetector(w io.Writer) *detector { - return &detector{Writer: w} +func newReader(data []byte, err error) *errReader { + return &errReader{data: data, err: err} } -func (d *detector) Write(data []byte) (int, error) { - d.Once.Do(func() { - d.contentType = http.DetectContentType(data) - }) - return d.Writer.Write(data) +func (r *errReader) Read(b []byte) (int, error) { + if r.offset >= len(r.data) { + return 0, io.EOF + } + n := copy(b, r.data[r.offset:]) + r.offset += n + if r.offset >= len(r.data) { + return n, r.err + } + return n, nil +} + +const contentTypeDetectSize = 512 + +func newDetector() *detector { + return &detector{done: make(chan struct{}), data: make([]byte, contentTypeDetectSize)} +} + +func (d *detector) Wait() { + <-d.done +} + +func (d *detector) SetReader(reader io.Reader) { + d.Reader = reader +} + +func (d *detector) Detect() { + n, err := d.Reader.Read(d.data) + if err != nil && err != io.EOF { + d.err = err + return + } + d.data = d.data[:n] + d.contentType = http.DetectContentType(d.data) + close(d.done) +} + +func (d *detector) MultiReader() io.Reader { + return io.MultiReader(newReader(d.data, d.err), d.Reader) } func isValidToken(s string) bool { @@ -84,10 +125,13 @@ func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address) r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) return } - writer := newDetector(r.Response.BodyWriter()) + readDetector := newDetector() options := new(client.GetObjectParams). WithAddress(objectAddress). - WithPayloadWriter(writer) + WithPayloadReaderHandler(func(reader io.Reader) { + readDetector.SetReader(reader) + readDetector.Detect() + }) obj, err = clnt.GetObject( r.RequestCtx, @@ -119,7 +163,9 @@ func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address) if r.Request.URI().QueryArgs().GetBool("download") { dis = "attachment" } + r.Response.SetBodyStream(readDetector.MultiReader(), int(obj.PayloadSize())) r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) + var contentType string for _, attr := range obj.Attributes() { key := attr.Key() val := attr.Value() @@ -142,13 +188,24 @@ func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address) r.Response.Header.Set("Last-Modified", time.Unix(value, 0).Format(time.RFC1123)) case object.AttributeContentType: - writer.contentType = val + contentType = val } } r.Response.Header.Set("x-object-id", obj.ID().String()) r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) r.Response.Header.Set("x-container-id", obj.ContainerID().String()) - r.SetContentType(writer.contentType) + + if len(contentType) == 0 { + if readDetector.err != nil { + r.log.Error("could not read object", zap.Error(err)) + r.Error("could not read object", fasthttp.StatusBadRequest) + return + } + readDetector.Wait() + contentType = readDetector.contentType + } + r.SetContentType(contentType) + r.Response.Header.Set("Content-Disposition", dis+"; filename="+path.Base(filename)) } diff --git a/downloader/reader_test.go b/downloader/reader_test.go new file mode 100644 index 0000000..a88795d --- /dev/null +++ b/downloader/reader_test.go @@ -0,0 +1,43 @@ +package downloader + +import ( + "fmt" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReader(t *testing.T) { + data := []byte("test string") + err := fmt.Errorf("something wrong") + + for _, tc := range []struct { + err error + buff []byte + }{ + {err: nil, buff: make([]byte, len(data)+1)}, + {err: nil, buff: make([]byte, len(data))}, + {err: nil, buff: make([]byte, len(data)-1)}, + {err: err, buff: make([]byte, len(data)+1)}, + {err: err, buff: make([]byte, len(data))}, + {err: err, buff: make([]byte, len(data)-1)}, + } { + var res []byte + var err error + var n int + + r := newReader(data, tc.err) + for err == nil { + n, err = r.Read(tc.buff) + res = append(res, tc.buff[:n]...) + } + + if tc.err == nil { + require.Equal(t, io.EOF, err) + } else { + require.Equal(t, tc.err, err) + } + require.Equal(t, data, res) + } +} diff --git a/go.mod b/go.mod index 78d6162..4eb2e41 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/fasthttp/router v1.3.5 github.com/mr-tron/base58 v1.1.3 // indirect github.com/nspcc-dev/neo-go v0.95.3 - github.com/nspcc-dev/neofs-api-go v1.27.1 + github.com/nspcc-dev/neofs-api-go v1.27.2-0.20210623111558-6d531a07a53d github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b github.com/prometheus/client_golang v1.9.0 github.com/prometheus/common v0.15.0 diff --git a/go.sum b/go.sum index c053121..5f5a1be 100644 --- a/go.sum +++ b/go.sum @@ -329,8 +329,9 @@ github.com/nspcc-dev/neo-go v0.95.3 h1:RxBKcmmatbSM2cETGhv3ritmrkU0gUnWItNZvtrBt github.com/nspcc-dev/neo-go v0.95.3/go.mod h1:t15xRFDVhz5o/pstptdoW9N9JJBNn1hZ6APMNiC6MrY= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-api-go v1.27.0/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= -github.com/nspcc-dev/neofs-api-go v1.27.1 h1:ONdKOnm0/hK6m38VTUliCHY6RTxg+IpAzY4G+BeOZG4= github.com/nspcc-dev/neofs-api-go v1.27.1/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= +github.com/nspcc-dev/neofs-api-go v1.27.2-0.20210623111558-6d531a07a53d h1:e8F3ijxrAJSf2jgJKFtgp/nay4Tx5EpZ0vFiKmDfosI= +github.com/nspcc-dev/neofs-api-go v1.27.2-0.20210623111558-6d531a07a53d/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= From 5b6cc990c0e20f8f624f17f0b07d2880f49eceaa Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 28 Jun 2021 13:24:46 +0300 Subject: [PATCH 198/548] [#71] Fix yaml config support Signed-off-by: Denis Kirillov --- README.md | 38 +++++++++++++++++++++++++++++++++++++- settings.go | 22 +++++++++++----------- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 17a5735..87d4788 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ $ HTTP_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 neofs-http-gw In general, everything available as CLI parameter can also be specified via environment variables, so they're not specifically mentioned in most cases -(see `--help` also). +(see `--help` also). If you prefer a config file you can use it in yaml format. ### Nodes and weights @@ -171,6 +171,42 @@ HTTP_GW_LOGGER_SAMPLING_THEREAFTER=int - Logger sampling thereafter HTTP_GW_LOGGER_TRACE_LEVEL=string - Logger show trace on level ``` +### Yaml file +Configuration file is optional and can be used instead of environment variables/other parameters. +It can be specified with `--config` parameter: +``` +$ neofs-http-gw --config your-config.yaml +``` + +Configuration file example: +``` +listen_address: 0.0.0.0:8082 + +wallet: + passphrase: 123456 + +logger: + level: debug + +peers: + 0: + address: grpc://s01.neofs.devenv:8080 + weight: 1 +``` + +To know nesting level of variable you need to cut off the prefix `HTTP_GW` from variable and split the rest parts by `_`. +For example variable `HTTP_GW_PEERS_0_WEIGHT=1` will be transformed to: +``` +peers: + 0: + weight: 1 +``` + +If parameter doesn't support environment variable (e.g. `--listen_address 0.0.0.0:8082`) form it is used as is: +``` +listen_address: 0.0.0.0:8082 +``` + ## HTTP API provided This gateway intentionally provides limited feature set and doesn't try to diff --git a/settings.go b/settings.go index 983ac84..ceffec6 100644 --- a/settings.go +++ b/settings.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "io" "os" "sort" "strconv" @@ -14,11 +13,7 @@ import ( "github.com/valyala/fasthttp" ) -type empty int - const ( - devNull = empty(0) - defaultRebalanceTimer = 15 * time.Second defaultRequestTimeout = 15 * time.Second defaultConnectTimeout = 30 * time.Second @@ -70,6 +65,7 @@ const ( cmdMetrics = "metrics" cmdWallet = "wallet" cmdAddress = "address" + cmdConfig = "config" ) var ignore = map[string]struct{}{ @@ -80,12 +76,11 @@ var ignore = map[string]struct{}{ cmdVersion: {}, } -func (empty) Read([]byte) (int, error) { return 0, io.EOF } - func settings() *viper.Viper { v := viper.New() v.AutomaticEnv() v.SetEnvPrefix(Prefix) + v.AllowEmptyEnv(true) v.SetConfigType("yaml") v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) @@ -102,6 +97,7 @@ func settings() *viper.Viper { flags.StringP(cmdWallet, "w", "", `path to the wallet`) flags.String(cmdAddress, "", `address of wallet account`) + config := flags.String(cmdConfig, "", "config path") flags.Bool(cmdVerbose, false, "debug gRPC connections") flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout") flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout") @@ -142,10 +138,6 @@ func settings() *viper.Viper { panic(err) } - if err := v.ReadConfig(devNull); err != nil { - panic(err) - } - if err := flags.Parse(os.Args); err != nil { panic(err) } @@ -183,6 +175,14 @@ func settings() *viper.Viper { os.Exit(0) } + if v.IsSet(cmdConfig) { + if cfgFile, err := os.Open(*config); err != nil { + panic(err) + } else if err := v.ReadConfig(cfgFile); err != nil { + panic(err) + } + } + if peers != nil && len(*peers) > 0 { for i := range *peers { v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".address", (*peers)[i]) From dee182a0c13a534b0f07a40602625a106d63ac9a Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 29 Jun 2021 12:25:14 +0300 Subject: [PATCH 199/548] downloader: fix Last-Modified header It should never ever be sent in local time, instead of Last-Modified: Tue, 29 Jun 2021 12:02:57 MSK one should receive Last-Modified: Tue, 29 Jun 2021 09:02:57 GMT Signed-off-by: Roman Khimov --- downloader/download.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/downloader/download.go b/downloader/download.go index 079244a..1272777 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -186,7 +186,7 @@ func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address) continue } r.Response.Header.Set("Last-Modified", - time.Unix(value, 0).Format(time.RFC1123)) + time.Unix(value, 0).UTC().Format(http.TimeFormat)) case object.AttributeContentType: contentType = val } From 7d8b89b8507ede52692691753451ff0573e374e4 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 29 Jun 2021 17:01:05 +0300 Subject: [PATCH 200/548] CHANGELOG: release v0.16.0 --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c699f2a..de5533f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ This document outlines major changes between releases. +## 0.16.0 (29 Jun 2021) + +We update HTTP gateway with NEP-6 wallets support, YAML configuration files +and small fixes. + +New features: + * YAML configuration file (#71) + +Behavior changes: + * gateway key needs to be stored in a proper NEP-6 wallet now, `-k` option is + no longer available, see `-w` and `-a` (#68) + +Bugs fixed: + * downloads were not streamed leading to excessive memory usage (#67) + * Last-Modified header incorrectly used local time (#75) + ## 0.15.2 (22 Jun 2021) New features: From 9241156af4be60c999dbe4ec24ee1c0c95f56f05 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 29 Jun 2021 17:19:30 +0300 Subject: [PATCH 201/548] README: fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 87d4788..f4894da 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ version Show current version Or you can also use a [Docker image](https://hub.docker.com/r/nspccdev/neofs-http-gw) provided for released -(and occasionaly unreleased) versions of gateway (`:latest` points to the +(and occasionally unreleased) versions of gateway (`:latest` points to the latest stable release). ## Execution From 2b8aa4914e655eb717086b9ed99116aa967941b9 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 9 Jul 2021 12:49:08 +0300 Subject: [PATCH 202/548] [#77] Added requests logging Signed-off-by: Denis Kirillov --- app.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/app.go b/app.go index 0fbc312..2e53a13 100644 --- a/app.go +++ b/app.go @@ -197,11 +197,11 @@ func (a *app) Serve(ctx context.Context) { // Configure router. r := router.New() r.RedirectTrailingSlash = true - r.POST("/upload/{cid}", uploader.Upload) + r.POST("/upload/{cid}", a.logger(uploader.Upload)) a.log.Info("added path /upload/{cid}") - r.GET("/get/{cid}/{oid}", downloader.DownloadByAddress) + r.GET("/get/{cid}/{oid}", a.logger(downloader.DownloadByAddress)) a.log.Info("added path /get/{cid}/{oid}") - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", downloader.DownloadByAttribute) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.DownloadByAttribute)) a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") // enable metrics if a.cfg.GetBool(cmdMetrics) { @@ -229,3 +229,14 @@ func (a *app) Serve(ctx context.Context) { a.log.Fatal("could not start server", zap.Error(err)) } } + +func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { + return fasthttp.RequestHandler(func(ctx *fasthttp.RequestCtx) { + a.log.Info("request", zap.String("remote", ctx.RemoteAddr().String()), + zap.ByteString("method", ctx.Method()), + zap.ByteString("path", ctx.Path()), + zap.ByteString("query", ctx.QueryArgs().QueryString()), + zap.Uint64("id", ctx.ID())) + h(ctx) + }) +} From d856fdf4c4ed802202c536c7bee97a6ef999e654 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 9 Jul 2021 16:28:39 +0300 Subject: [PATCH 203/548] [#76] Added HEAD methods Signed-off-by: Denis Kirillov --- app.go | 2 + downloader/download.go | 199 +++++++++++++++++++++++++++++++---------- 2 files changed, 154 insertions(+), 47 deletions(-) diff --git a/app.go b/app.go index 2e53a13..52991ff 100644 --- a/app.go +++ b/app.go @@ -200,8 +200,10 @@ func (a *app) Serve(ctx context.Context) { r.POST("/upload/{cid}", a.logger(uploader.Upload)) a.log.Info("added path /upload/{cid}") r.GET("/get/{cid}/{oid}", a.logger(downloader.DownloadByAddress)) + r.HEAD("/get/{cid}/{oid}", a.logger(downloader.HeadByAddress)) a.log.Info("added path /get/{cid}/{oid}") r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.DownloadByAttribute)) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.HeadByAttribute)) a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") // enable metrics if a.cfg.GetBool(cmdMetrics) { diff --git a/downloader/download.go b/downloader/download.go index 1272777..0b77969 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -45,6 +45,10 @@ type ( } ) +var errObjectNotFound = errors.New("object not found") + +const sizeToDetectType = 512 + func newReader(data []byte, err error) *errReader { return &errReader{data: data, err: err} } @@ -112,7 +116,7 @@ func isValidValue(s string) bool { return true } -func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address) { +func (r request) receiveFile(clnt client.Object, objectAddress *object.Address) { var ( err error dis = "inline" @@ -133,31 +137,9 @@ func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address) readDetector.Detect() }) - obj, err = clnt.GetObject( - r.RequestCtx, - options, - ) + obj, err = clnt.GetObject(r.RequestCtx, options, bearerOpts(r.RequestCtx)) if err != nil { - r.log.Error( - "could not receive object", - zap.Stringer("elapsed", time.Since(start)), - zap.Error(err), - ) - var ( - msg = fmt.Sprintf("could not receive object: %v", err) - code = fasthttp.StatusBadRequest - cause = err - ) - for unwrap := errors.Unwrap(err); unwrap != nil; unwrap = errors.Unwrap(cause) { - cause = unwrap - } - if st, ok := status.FromError(cause); ok && st != nil { - if st.Code() == codes.NotFound { - code = fasthttp.StatusNotFound - } - msg = st.Message() - } - r.Error(msg, code) + r.handleNeoFSErr(err, start) return } if r.Request.URI().QueryArgs().GetBool("download") { @@ -209,6 +191,99 @@ func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address) r.Response.Header.Set("Content-Disposition", dis+"; filename="+path.Base(filename)) } +func bearerOpts(ctx context.Context) client.CallOption { + if tkn, err := tokens.LoadBearerToken(ctx); err == nil { + return client.WithBearer(tkn) + } + return client.WithBearer(nil) +} + +func (r request) headObject(clnt client.Object, objectAddress *object.Address) { + var start = time.Now() + if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { + r.log.Error("could not fetch and store bearer token", zap.Error(err)) + r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) + return + } + + options := new(client.ObjectHeaderParams).WithAddress(objectAddress) + bearerOpt := bearerOpts(r.RequestCtx) + obj, err := clnt.GetObjectHeader(r.RequestCtx, options, bearerOpt) + if err != nil { + r.handleNeoFSErr(err, start) + return + } + + r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) + var contentType string + for _, attr := range obj.Attributes() { + key := attr.Key() + val := attr.Value() + if !isValidToken(key) || !isValidValue(val) { + continue + } + r.Response.Header.Set("X-Attribute-"+key, val) + switch key { + case object.AttributeTimestamp: + value, err := strconv.ParseInt(val, 10, 64) + if err != nil { + r.log.Info("couldn't parse creation date", + zap.String("key", key), + zap.String("val", val), + zap.Error(err)) + continue + } + r.Response.Header.Set("Last-Modified", time.Unix(value, 0).UTC().Format(http.TimeFormat)) + case object.AttributeContentType: + contentType = val + } + } + r.Response.Header.Set("x-object-id", obj.ID().String()) + r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) + r.Response.Header.Set("x-container-id", obj.ContainerID().String()) + + if len(contentType) == 0 { + objRange := object.NewRange() + objRange.SetOffset(0) + if sizeToDetectType < obj.PayloadSize() { + objRange.SetLength(sizeToDetectType) + } else { + objRange.SetLength(obj.PayloadSize()) + } + ops := new(client.RangeDataParams).WithAddress(objectAddress).WithRange(objRange) + data, err := clnt.ObjectPayloadRangeData(r.RequestCtx, ops, bearerOpt) + if err != nil { + r.handleNeoFSErr(err, start) + return + } + contentType = http.DetectContentType(data) + } + r.SetContentType(contentType) +} + +func (r *request) handleNeoFSErr(err error, start time.Time) { + r.log.Error( + "could not receive object", + zap.Stringer("elapsed", time.Since(start)), + zap.Error(err), + ) + var ( + msg = fmt.Sprintf("could not receive object: %v", err) + code = fasthttp.StatusBadRequest + cause = err + ) + for unwrap := errors.Unwrap(err); unwrap != nil; unwrap = errors.Unwrap(cause) { + cause = unwrap + } + if st, ok := status.FromError(cause); ok && st != nil { + if st.Code() == codes.NotFound { + code = fasthttp.StatusNotFound + } + msg = st.Message() + } + r.Error(msg, code) +} + func (o objectIDs) Slice() []string { res := make([]string, 0, len(o)) for _, oid := range o { @@ -242,53 +317,84 @@ func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *requ // DownloadByAddress handles download requests using simple cid/oid format. func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { + d.byAddress(c, request.receiveFile) +} + +// HeadByAddress handles head requests using simple cid/oid format. +func (d *Downloader) HeadByAddress(c *fasthttp.RequestCtx) { + d.byAddress(c, request.headObject) +} + +// byAddress is wrapper for function (e.g. request.headObject, request.receiveFile) that +// prepares request and object address to it. +func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) { var ( - err error address = object.NewAddress() cid, _ = c.UserValue("cid").(string) oid, _ = c.UserValue("oid").(string) val = strings.Join([]string{cid, oid}, "/") log = d.log.With(zap.String("cid", cid), zap.String("oid", oid)) ) - if err = address.Parse(val); err != nil { + if err := address.Parse(val); err != nil { log.Error("wrong object address", zap.Error(err)) c.Error("wrong object address", fasthttp.StatusBadRequest) return } - d.newRequest(c, log).receiveFile(d.pool, address) + f(*d.newRequest(c, log), d.pool, address) } // DownloadByAttribute handles attribute-based download requests. func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { + d.byAttribute(c, request.receiveFile) +} + +// HeadByAttribute handles attribute-based head requests. +func (d *Downloader) HeadByAttribute(c *fasthttp.RequestCtx) { + d.byAttribute(c, request.headObject) +} + +// byAttribute is wrapper similar to byAddress. +func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) { var ( - err error - scid, _ = c.UserValue("cid").(string) - key, _ = c.UserValue("attr_key").(string) - val, _ = c.UserValue("attr_val").(string) - log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) - ids []*object.ID + httpStatus = fasthttp.StatusBadRequest + scid, _ = c.UserValue("cid").(string) + key, _ = c.UserValue("attr_key").(string) + val, _ = c.UserValue("attr_val").(string) + log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ) - cid := cid.New() - if err = cid.Parse(scid); err != nil { + containerID := cid.New() + if err := containerID.Parse(scid); err != nil { log.Error("wrong container id", zap.Error(err)) - c.Error("wrong container id", fasthttp.StatusBadRequest) + c.Error("wrong container id", httpStatus) return } + address, err := d.searchObject(c, log, containerID, key, val) + if err != nil { + log.Error("couldn't search object", zap.Error(err)) + if errors.Is(err, errObjectNotFound) { + httpStatus = fasthttp.StatusNotFound + } + c.Error("couldn't search object", httpStatus) + return + } + + f(*d.newRequest(c, log), d.pool, address) +} + +func (d *Downloader) searchObject(c *fasthttp.RequestCtx, log *zap.Logger, cid *cid.ID, key, val string) (*object.Address, error) { options := object.NewSearchFilters() options.AddRootFilter() options.AddFilter(key, val, object.MatchStringEqual) sops := new(client.SearchObjectParams).WithContainerID(cid).WithSearchFilters(options) - if ids, err = d.pool.SearchObject(c, sops); err != nil { - log.Error("something went wrong", zap.Error(err)) - c.Error("something went wrong", fasthttp.StatusBadRequest) - return - } else if len(ids) == 0 { - log.Debug("object not found") - c.Error("object not found", fasthttp.StatusNotFound) - return + ids, err := d.pool.SearchObject(c, sops) + if err != nil { + return nil, err + } + if len(ids) == 0 { + return nil, errObjectNotFound } if len(ids) > 1 { log.Debug("found multiple objects", @@ -298,6 +404,5 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { address := object.NewAddress() address.SetContainerID(cid) address.SetObjectID(ids[0]) - - d.newRequest(c, log).receiveFile(d.pool, address) + return address, nil } From 87918483c6c7da2fe322b3345fba6b972aef6e2d Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 15 Jul 2021 10:57:22 +0300 Subject: [PATCH 204/548] [#76] Refactor HeadObject logic Signed-off-by: Denis Kirillov --- README.md | 4 +- downloader/download.go | 75 ----------------------------------- downloader/head.go | 88 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 76 deletions(-) create mode 100644 downloader/head.go diff --git a/README.md b/README.md index f4894da..57a1504 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,8 @@ $ neofs-cli -r 192.168.130.72:8080 -k 6PYLKJhiSub5imt6WCVy6Quxtd9xu176omev1vWYov #### Requests +The following requests support GET/HEAD methods. + ##### By IDs Basic downloading involves container ID and object ID and is done via GET @@ -311,7 +313,7 @@ $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m #### Replies -You get object contents in the reply body, but at the same time you also get a +You get object contents in the reply body (if GET method was used), but at the same time you also get a set of reply headers generated using the following rules: * `Content-Length` is set to the length of the object * `Content-Type` is autodetected dynamically by gateway diff --git a/downloader/download.go b/downloader/download.go index 0b77969..87a1094 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -47,8 +47,6 @@ type ( var errObjectNotFound = errors.New("object not found") -const sizeToDetectType = 512 - func newReader(data []byte, err error) *errReader { return &errReader{data: data, err: err} } @@ -198,69 +196,6 @@ func bearerOpts(ctx context.Context) client.CallOption { return client.WithBearer(nil) } -func (r request) headObject(clnt client.Object, objectAddress *object.Address) { - var start = time.Now() - if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { - r.log.Error("could not fetch and store bearer token", zap.Error(err)) - r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) - return - } - - options := new(client.ObjectHeaderParams).WithAddress(objectAddress) - bearerOpt := bearerOpts(r.RequestCtx) - obj, err := clnt.GetObjectHeader(r.RequestCtx, options, bearerOpt) - if err != nil { - r.handleNeoFSErr(err, start) - return - } - - r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) - var contentType string - for _, attr := range obj.Attributes() { - key := attr.Key() - val := attr.Value() - if !isValidToken(key) || !isValidValue(val) { - continue - } - r.Response.Header.Set("X-Attribute-"+key, val) - switch key { - case object.AttributeTimestamp: - value, err := strconv.ParseInt(val, 10, 64) - if err != nil { - r.log.Info("couldn't parse creation date", - zap.String("key", key), - zap.String("val", val), - zap.Error(err)) - continue - } - r.Response.Header.Set("Last-Modified", time.Unix(value, 0).UTC().Format(http.TimeFormat)) - case object.AttributeContentType: - contentType = val - } - } - r.Response.Header.Set("x-object-id", obj.ID().String()) - r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) - r.Response.Header.Set("x-container-id", obj.ContainerID().String()) - - if len(contentType) == 0 { - objRange := object.NewRange() - objRange.SetOffset(0) - if sizeToDetectType < obj.PayloadSize() { - objRange.SetLength(sizeToDetectType) - } else { - objRange.SetLength(obj.PayloadSize()) - } - ops := new(client.RangeDataParams).WithAddress(objectAddress).WithRange(objRange) - data, err := clnt.ObjectPayloadRangeData(r.RequestCtx, ops, bearerOpt) - if err != nil { - r.handleNeoFSErr(err, start) - return - } - contentType = http.DetectContentType(data) - } - r.SetContentType(contentType) -} - func (r *request) handleNeoFSErr(err error, start time.Time) { r.log.Error( "could not receive object", @@ -320,11 +255,6 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { d.byAddress(c, request.receiveFile) } -// HeadByAddress handles head requests using simple cid/oid format. -func (d *Downloader) HeadByAddress(c *fasthttp.RequestCtx) { - d.byAddress(c, request.headObject) -} - // byAddress is wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) { @@ -349,11 +279,6 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { d.byAttribute(c, request.receiveFile) } -// HeadByAttribute handles attribute-based head requests. -func (d *Downloader) HeadByAttribute(c *fasthttp.RequestCtx) { - d.byAttribute(c, request.headObject) -} - // byAttribute is wrapper similar to byAddress. func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) { var ( diff --git a/downloader/head.go b/downloader/head.go new file mode 100644 index 0000000..5ecfd19 --- /dev/null +++ b/downloader/head.go @@ -0,0 +1,88 @@ +package downloader + +import ( + "net/http" + "strconv" + "time" + + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-http-gw/tokens" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +const sizeToDetectType = 512 + +func (r request) headObject(clnt client.Object, objectAddress *object.Address) { + var start = time.Now() + if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { + r.log.Error("could not fetch and store bearer token", zap.Error(err)) + r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) + return + } + + options := new(client.ObjectHeaderParams).WithAddress(objectAddress) + bearerOpt := bearerOpts(r.RequestCtx) + obj, err := clnt.GetObjectHeader(r.RequestCtx, options, bearerOpt) + if err != nil { + r.handleNeoFSErr(err, start) + return + } + + r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) + var contentType string + for _, attr := range obj.Attributes() { + key := attr.Key() + val := attr.Value() + if !isValidToken(key) || !isValidValue(val) { + continue + } + r.Response.Header.Set("X-Attribute-"+key, val) + switch key { + case object.AttributeTimestamp: + value, err := strconv.ParseInt(val, 10, 64) + if err != nil { + r.log.Info("couldn't parse creation date", + zap.String("key", key), + zap.String("val", val), + zap.Error(err)) + continue + } + r.Response.Header.Set("Last-Modified", time.Unix(value, 0).UTC().Format(http.TimeFormat)) + case object.AttributeContentType: + contentType = val + } + } + r.Response.Header.Set("x-object-id", obj.ID().String()) + r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) + r.Response.Header.Set("x-container-id", obj.ContainerID().String()) + + if len(contentType) == 0 { + objRange := object.NewRange() + objRange.SetOffset(0) + if sizeToDetectType < obj.PayloadSize() { + objRange.SetLength(sizeToDetectType) + } else { + objRange.SetLength(obj.PayloadSize()) + } + ops := new(client.RangeDataParams).WithAddress(objectAddress).WithRange(objRange) + data, err := clnt.ObjectPayloadRangeData(r.RequestCtx, ops, bearerOpt) + if err != nil { + r.handleNeoFSErr(err, start) + return + } + contentType = http.DetectContentType(data) + } + r.SetContentType(contentType) +} + +// HeadByAddress handles head requests using simple cid/oid format. +func (d *Downloader) HeadByAddress(c *fasthttp.RequestCtx) { + d.byAddress(c, request.headObject) +} + +// HeadByAttribute handles attribute-based head requests. +func (d *Downloader) HeadByAttribute(c *fasthttp.RequestCtx) { + d.byAttribute(c, request.headObject) +} From 03a3f986ffb63184fec97ba0faefd7821f946f4e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 20 Jul 2021 13:40:39 +0300 Subject: [PATCH 205/548] [#30] Fix not found responses Signed-off-by: Denis Kirillov --- downloader/download.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 87a1094..cd5e87c 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -18,8 +18,6 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) type ( @@ -210,12 +208,13 @@ func (r *request) handleNeoFSErr(err error, start time.Time) { for unwrap := errors.Unwrap(err); unwrap != nil; unwrap = errors.Unwrap(cause) { cause = unwrap } - if st, ok := status.FromError(cause); ok && st != nil { - if st.Code() == codes.NotFound { - code = fasthttp.StatusNotFound - } - msg = st.Message() + + if strings.Contains(cause.Error(), "not found") || + strings.Contains(cause.Error(), "can't fetch container info") { + code = fasthttp.StatusNotFound + msg = errObjectNotFound.Error() } + r.Error(msg, code) } From 67ea967ec8bb18d603370909cc2fa0f6bf7e827a Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Fri, 23 Jul 2021 20:09:47 +0300 Subject: [PATCH 206/548] Update dependencies versions Signed-off-by: Angira Kekteeva --- go.mod | 43 ++-- go.sum | 696 +++++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 474 insertions(+), 265 deletions(-) diff --git a/go.mod b/go.mod index 4eb2e41..3774c86 100644 --- a/go.mod +++ b/go.mod @@ -3,23 +3,32 @@ module github.com/nspcc-dev/neofs-http-gw go 1.16 require ( - github.com/fasthttp/router v1.3.5 - github.com/mr-tron/base58 v1.1.3 // indirect - github.com/nspcc-dev/neo-go v0.95.3 - github.com/nspcc-dev/neofs-api-go v1.27.2-0.20210623111558-6d531a07a53d - github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b - github.com/prometheus/client_golang v1.9.0 - github.com/prometheus/common v0.15.0 + github.com/DataDog/zstd v1.4.8 // indirect + github.com/andybalholm/brotli v1.0.3 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/dgraph-io/badger/v2 v2.2007.3 // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect + github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/fasthttp/router v1.4.1 + github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/klauspost/compress v1.13.1 // indirect + github.com/nspcc-dev/neo-go v0.96.1 + github.com/nspcc-dev/neofs-api-go v1.28.3 + github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210707142403-393034fd73bb + github.com/prometheus/client_golang v1.11.0 + github.com/prometheus/common v0.29.0 + github.com/prometheus/procfs v0.7.1 // indirect github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.7.1 + github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 - github.com/valyala/fasthttp v1.22.0 - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/zap v1.16.0 - golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect - golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect - golang.org/x/tools v0.0.0-20200123022218-593de606220b // indirect - google.golang.org/grpc v1.36.1 + github.com/valyala/fasthttp v1.28.0 + go.uber.org/multierr v1.7.0 // indirect + go.uber.org/zap v1.18.1 + golang.org/x/net v0.0.0-20210716203947-853a461950ff // indirect + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect + google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f // indirect + google.golang.org/grpc v1.39.0 ) - -replace github.com/valyala/fasthttp => github.com/nspcc-dev/fasthttp v1.19.1-0.20210428122823-ab82e78c7994 diff --git a/go.sum b/go.sum index 5f5a1be..2fa3f49 100644 --- a/go.sum +++ b/go.sum @@ -5,72 +5,97 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1:MJCkWUBhi9pn/CrYO1Q3P687y2KeahrOPS9BD9LDGb0= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= -github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= +github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= +github.com/Workiva/go-datastructures v1.0.53/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= +github.com/abiosoft/ishell/v2 v2.0.2/go.mod h1:E4oTCXfo6QjoCart0QYa5m9w4S+deXs/P/9jA77A9Bs= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/alecthomas/participle v0.7.1/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY= -github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 h1:45bxf7AZMwWcqkLzDAQugVEwedisr5nRJ1r+7LYnv0U= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= -github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= -github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/alicebob/miniredis/v2 v2.15.1 h1:Fw+ixAJPmKhCLBqDwHlTDqxUxp0xjEwXczEpt1B6r7k= +github.com/alicebob/miniredis/v2 v2.15.1/go.mod h1:gquAfGbzn92jvtrSC69+6zZnwSODVXVpYDRaGhWaL6I= +github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= +github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= +github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -80,90 +105,94 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI= +github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= -github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU= +github.com/dgraph-io/badger/v2 v2.2007.3 h1:Sl9tQWz92WCbVSe8pj04Tkqlm2boW+KAxd+XSs58SQI= +github.com/dgraph-io/badger/v2 v2.2007.3/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/fasthttp/router v1.3.5 h1:XGZZtcYUH+9ZoLxKhwF66F8sGnMT+Jg00j0JdVzwmDA= -github.com/fasthttp/router v1.3.5/go.mod h1:BylQKgvh6YQkR0mvL60+HJyTaGwcn5d8UFNweOb/Nw8= +github.com/fasthttp/router v1.4.1 h1:3xPUO+hy/HAkgGDSd5sX5w18cyGDIFbC7vip8KwPDk8= +github.com/fasthttp/router v1.4.1/go.mod h1:4P0Kq4C882tA2evBKDW7De7hGfWmvV8FN+zqt8Lu49Q= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-redis/redis v6.10.2+incompatible h1:SLbqrO/Ik1nhXA5/cbEs1P5MUBo1Qq4ihlNfGnnipPw= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= +github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -171,13 +200,16 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -185,35 +217,43 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -224,7 +264,6 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -236,32 +275,33 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.11.8 h1:difgzQsp5mdAz9v8lm3P/I+EpDKMU/6uTMw1y1FObuo= -github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= +github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -269,21 +309,18 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -294,138 +331,110 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= -github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E= github.com/nspcc-dev/dbft v0.0.0-20210302103605-cc75991b7cfb/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= -github.com/nspcc-dev/fasthttp v1.19.1-0.20210428122823-ab82e78c7994 h1:FBzAFRxTVcvty6csu0ujCJbEChOK9MC0GW9FNvKggqw= -github.com/nspcc-dev/fasthttp v1.19.1-0.20210428122823-ab82e78c7994/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= +github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= github.com/nspcc-dev/neo-go v0.95.1/go.mod h1:bW07ge1WFXsBgqrcPpLUr6OcyQxHqM26MZNesWMdH0c= -github.com/nspcc-dev/neo-go v0.95.3 h1:RxBKcmmatbSM2cETGhv3ritmrkU0gUnWItNZvtrBtI0= github.com/nspcc-dev/neo-go v0.95.3/go.mod h1:t15xRFDVhz5o/pstptdoW9N9JJBNn1hZ6APMNiC6MrY= +github.com/nspcc-dev/neo-go v0.96.1 h1:JaKWvM/vvQ48bq2ADNj7zH/6Ek38Iqxo22hdu2lhxmY= +github.com/nspcc-dev/neo-go v0.96.1/go.mod h1:yguwQBpWMTHx07INKoElJT8Gga1LUdTSi0gT75ywJ68= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= -github.com/nspcc-dev/neofs-api-go v1.27.0/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= github.com/nspcc-dev/neofs-api-go v1.27.1/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= -github.com/nspcc-dev/neofs-api-go v1.27.2-0.20210623111558-6d531a07a53d h1:e8F3ijxrAJSf2jgJKFtgp/nay4Tx5EpZ0vFiKmDfosI= -github.com/nspcc-dev/neofs-api-go v1.27.2-0.20210623111558-6d531a07a53d/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= +github.com/nspcc-dev/neofs-api-go v1.28.3 h1:53Ec3hv3LtI3uuG1H8Yp2OKOIdsAqQVfUOyClhnhc9g= +github.com/nspcc-dev/neofs-api-go v1.28.3/go.mod h1:YRIzUqBj/lGbmFm8mmCh54ZOzcJKkEIhv2s7ZvSLv3M= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b h1:l99sCKR/mt+iFb4p1836qtoXUQEGYlEzqWAeAliEaE8= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210615074944-86a9aa92599b/go.mod h1:1djNrOkpTTbNUlJM/MvTmohJUaWKUMy9JHSCCA8rJEc= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210707142403-393034fd73bb h1:DYSMccPlDeYE41ESLgqWu8lo3Jl1B1Ldudt+zp445uc= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210707142403-393034fd73bb/go.mod h1:I2V7pAvSlW48awy3B1CyBo/j4oE/Q56ghieWh46FWZc= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU= -github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= -github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE= +github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.1 h1:TlEtJq5GvGqMykEwWzbZWjjztF86swFhsPix1i0bkgA= +github.com/prometheus/procfs v0.7.1/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/savsgio/gotils v0.0.0-20210105085219-0567298fdcac h1:5pr1F6tcjeIKPKlrQZW6hzB0WxZge7SkkYVGG/e5pLY= -github.com/savsgio/gotils v0.0.0-20210105085219-0567298fdcac/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= +github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 h1:N3Af8f13ooDKcIhsmFT7Z05CStZWu4C7Md0uDEy4q6o= +github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873/go.mod h1:dmPawKuiAeG/aFYVs2i+Dyosoo7FNcm+Pi8iK6ZUrX8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -434,29 +443,25 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= -github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -468,49 +473,62 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73 h1:I2drr5K0tykBofr74ZEGliE/Hf6fNkEbcPyFvsy7wZk= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= +github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= +github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= +github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= +github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/valyala/fasthttp v1.28.0 h1:ruVmTmZaBR5i67NqnjvvH5gEv0zwHfWtbjoyW98iho4= +github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= -github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7 h1:Y17pEjKgx2X0A69WQPGa8hx/Myzu+4NdUxlkZpbAYio= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da h1:NimzV1aGyq29m5ukMK0AMWEhFaL/lrEOaephfuoiARg= +github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -519,16 +537,24 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -538,22 +564,31 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -562,28 +597,67 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210716203947-853a461950ff h1:j2EK/QoxYNBsXI4R7fQkkRUk8y6wnOBI+6hgPdP/6Ds= +golang.org/x/net v0.0.0-20210716203947-853a461950ff/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -598,37 +672,72 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -645,55 +754,144 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200123022218-593de606220b h1:ztSlcncMErSAUzXwnVO1iTPxHwtvOHBB26SGiyYXIEE= -golang.org/x/tools v0.0.0-20200123022218-593de606220b/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f h1:YORWxaStkWBnWgELOHTmDrqNlFXuVGEbhwbB5iK94bQ= +google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.36.1 h1:cmUfbeGKnz9+2DD/UYsMQXeqbHZqZDs4eQwW0sFOpBY= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -702,41 +900,43 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From eb6d46fe514c0b63f2accc57fb0cef8ed05e3aca Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 28 Jul 2021 12:52:17 +0300 Subject: [PATCH 207/548] [#82] Update sdk-go Signed-off-by: Denis Kirillov --- go.mod | 2 +- go.sum | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3774c86..bd8a573 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/klauspost/compress v1.13.1 // indirect github.com/nspcc-dev/neo-go v0.96.1 github.com/nspcc-dev/neofs-api-go v1.28.3 - github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210707142403-393034fd73bb + github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210728093755-d95d722d6156 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 github.com/prometheus/procfs v0.7.1 // indirect diff --git a/go.sum b/go.sum index 2fa3f49..466b97f 100644 --- a/go.sum +++ b/go.sum @@ -187,6 +187,8 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -366,8 +368,8 @@ github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9K github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210707142403-393034fd73bb h1:DYSMccPlDeYE41ESLgqWu8lo3Jl1B1Ldudt+zp445uc= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210707142403-393034fd73bb/go.mod h1:I2V7pAvSlW48awy3B1CyBo/j4oE/Q56ghieWh46FWZc= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210728093755-d95d722d6156 h1:HiCIiXmv9HEbJh6okyW/8OCLNn8r3AbvgmBUtKh+8wE= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210728093755-d95d722d6156/go.mod h1:NOjwgDKeU8ZoXO0k2cwMi5HV3BRJVJLLaXrYpVjF1LU= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= @@ -790,6 +792,7 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -859,6 +862,7 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= From 196cfb31376df2d3491ee03abf3c3c9bba07504a Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 28 Jul 2021 13:15:52 +0300 Subject: [PATCH 208/548] CHANGELOG: release v0.16.1 Signed-off-by: Denis Kirillov --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de5533f..1086100 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ This document outlines major changes between releases. +## 0.16.1 (28 Jul 2021) + +New features: +* logging requests (#77) +* HEAD methods for download routes (#76) + +Improvements: +* updated sdk-go dependency (#82) + +Bugs fixed: +* wrong NotFound status was used (#30) + ## 0.16.0 (29 Jun 2021) We update HTTP gateway with NEP-6 wallets support, YAML configuration files From 5caac8f5265607117a40e0bceff36ed5f78fc008 Mon Sep 17 00:00:00 2001 From: anastasia prasolova Date: Tue, 31 Aug 2021 18:02:04 +0300 Subject: [PATCH 209/548] [#86] Remove GitHub workflow of image publishing Signed-off-by: anastasia prasolova --- .github/workflows/publish_to_dockerhub.yml | 84 ---------------------- 1 file changed, 84 deletions(-) delete mode 100644 .github/workflows/publish_to_dockerhub.yml diff --git a/.github/workflows/publish_to_dockerhub.yml b/.github/workflows/publish_to_dockerhub.yml deleted file mode 100644 index 7a66b76..0000000 --- a/.github/workflows/publish_to_dockerhub.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Push images to DockerHub - -# Controls when the action will run. -on: - push: - # Publish `master` as Docker `latest` and `git_revision` images. - branches: - - master - release: - # Publish released commit as Docker `latest` and `git_revision` images. - types: - - published - - # Allows to run this workflow manually from the Actions tab. - workflow_dispatch: - inputs: - ref: - description: 'Ref to build Docker image [default: latest master; examples: v0.92.0, 0a4ff9d3e4a9ab432fd5812eb18c98e03b5a7432]' - required: false - default: '' - push_image: - description: 'Push image to DockerHub [default: false; examples: true, false]' - required: false - default: 'false' - -# A workflow run. -jobs: - publish: - name: Publish image to DockerHub - runs-on: ubuntu-20.04 - steps: - - name: Checkout (manual run) - if: ${{ github.event_name == 'workflow_dispatch' }} - uses: actions/checkout@v2 - with: - ref: ${{ github.event.inputs.ref }} - # Allows to fetch all history for all branches and tags. Need this for proper versioning. - fetch-depth: 0 - - - name: Checkout (automatical run) - if: ${{ github.event_name != 'workflow_dispatch' }} - uses: actions/checkout@v2 - with: - # Allows to fetch all history for all branches and tags. Need this for proper versioning. - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.16 - - - name: Restore go modules from cache - uses: actions/cache@v2 - with: - path: /home/runner/go/pkg/mod - key: deps-${{ hashFiles('go.sum') }} - - - name: Update Go modules - run: make dep - - - name: Build image - run: make image - - - name: Build image with 'latest' tag - if: ${{ github.event_name == 'release' && github.event.release.target_commitish == 'master' }} - run: make image - env: - HUB_TAG: latest - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Push image to registry - if: ${{ github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.push_image == 'true') }} - run: make image-push - - - name: Push image with 'latest' tag to registry - if: ${{ github.event_name == 'release' && github.event.release.target_commitish == 'master' }} - run: make image-push - env: - HUB_TAG: latest From d2b05329292881f987e71996023b52d3f7022058 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 6 Sep 2021 15:59:09 +0300 Subject: [PATCH 210/548] [#85] Add integration tests Signed-off-by: Denis Kirillov --- downloader/reader_test.go | 43 ++++ go.mod | 1 + go.sum | 431 +++++++++++++++++++++++++++++++++++++- integration_test.go | 259 +++++++++++++++++++++++ 4 files changed, 733 insertions(+), 1 deletion(-) create mode 100644 integration_test.go diff --git a/downloader/reader_test.go b/downloader/reader_test.go index a88795d..96a42e1 100644 --- a/downloader/reader_test.go +++ b/downloader/reader_test.go @@ -1,8 +1,10 @@ package downloader import ( + "bytes" "fmt" "io" + "strings" "testing" "github.com/stretchr/testify/require" @@ -41,3 +43,44 @@ func TestReader(t *testing.T) { require.Equal(t, data, res) } } + +func TestDetector(t *testing.T) { + txtContentType := "text/plain; charset=utf-8" + sb := strings.Builder{} + for i := 0; i < 10; i++ { + sb.WriteString("Some txt content. Content-Type must be detected properly by detector.") + } + + for _, tc := range []struct { + Name string + ContentType string + Expected string + }{ + { + Name: "less than 512b", + ContentType: txtContentType, + Expected: sb.String()[:256], + }, + { + Name: "more than 512b", + ContentType: txtContentType, + Expected: sb.String(), + }, + } { + t.Run(tc.Name, func(t *testing.T) { + detector := newDetector() + + go func() { + detector.SetReader(bytes.NewBufferString(tc.Expected)) + detector.Detect() + }() + + detector.Wait() + require.Equal(t, tc.ContentType, detector.contentType) + + data, err := io.ReadAll(detector.MultiReader()) + require.NoError(t, err) + require.Equal(t, tc.Expected, string(data)) + }) + } +} diff --git a/go.mod b/go.mod index bd8a573..4f7f1e7 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 + github.com/testcontainers/testcontainers-go v0.11.1 github.com/valyala/fasthttp v1.28.0 go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.18.1 diff --git a/go.sum b/go.sum index 466b97f..e588d4b 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -37,6 +38,19 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1:MJCkWUBhi9pn/CrYO1Q3P687y2KeahrOPS9BD9LDGb0= @@ -46,8 +60,29 @@ github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1 github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3 h1:mw6pDQqv38/WGF1cO/jF5t/jyAJ2yi7CmtFLLO5tGFI= +github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= +github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= +github.com/Microsoft/hcsshim v0.8.16 h1:8/auA4LFIZFTGrqfKhGBSXwM6/4X1fHa/xniyEHu8ac= +github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/Workiva/go-datastructures v1.0.53/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= @@ -59,6 +94,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= @@ -70,19 +106,28 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1 h1:zFRi26YWd7NIorBXe8UkevRl0dIvk/AnXHWaAiZG+Yk= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= @@ -96,35 +141,131 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= +github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= +github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 h1:hkGVFjz+plgr5UfxZUTPFbUFIF/Km6/s+RVRIRHLrrY= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4 h1:zjz4MOAOFgdBlwid2nNUlJ3YLpVi/97L36lfMYJex60= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= +github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e h1:6JKvHHt396/qabvMhnhUZvWaHZzfVfldxE60TK8YLhg= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= +github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= +github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= +github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= +github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= +github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= +github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= +github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= github.com/dgraph-io/badger/v2 v2.2007.3 h1:Sl9tQWz92WCbVSe8pj04Tkqlm2boW+KAxd+XSs58SQI= github.com/dgraph-io/badger/v2 v2.2007.3/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= @@ -132,11 +273,35 @@ github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KP github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= +github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= +github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -146,38 +311,68 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fasthttp/router v1.4.1 h1:3xPUO+hy/HAkgGDSd5sX5w18cyGDIFbC7vip8KwPDk8= github.com/fasthttp/router v1.4.1/go.mod h1:4P0Kq4C882tA2evBKDW7De7hGfWmvV8FN+zqt8Lu49Q= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= +github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -228,6 +423,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -243,6 +439,7 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -250,16 +447,27 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= +github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= @@ -280,9 +488,17 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -295,13 +511,17 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -309,23 +529,34 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -336,15 +567,32 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= +github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM= +github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= +github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE= +github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= @@ -375,26 +623,59 @@ github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYv github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc93 h1:x2UMpOOVf3kQ8arv/EsDGwim8PTNqzL1/EYDr/+scOM= +github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= +github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -402,30 +683,42 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.1 h1:TlEtJq5GvGqMykEwWzbZWjjztF86swFhsPix1i0bkgA= github.com/prometheus/procfs v0.7.1/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= @@ -434,38 +727,58 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 h1:N3Af8f13ooDKcIhsmFT7Z05CStZWu4C7Md0uDEy4q6o= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873/go.mod h1:dmPawKuiAeG/aFYVs2i+Dyosoo7FNcm+Pi8iK6ZUrX8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= +github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -475,14 +788,25 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/testcontainers/testcontainers-go v0.11.1 h1:FiYsB83LSGbiawoV8TpAZGfcCUbtaeeg1SXqEKUxh08= +github.com/testcontainers/testcontainers-go v0.11.1/go.mod h1:/V0UVq+1e7NWYoqTPog179clf0Qp9TOyp4EcXaEFQz8= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -492,6 +816,16 @@ github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfY github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= +github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -502,21 +836,29 @@ github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBU github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da h1:NimzV1aGyq29m5ukMK0AMWEhFaL/lrEOaephfuoiARg= github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -532,18 +874,23 @@ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -587,20 +934,27 @@ golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -621,6 +975,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= @@ -659,6 +1014,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -669,23 +1025,38 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -698,14 +1069,23 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -713,6 +1093,7 @@ golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -736,11 +1117,16 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -751,7 +1137,9 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -801,6 +1189,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -830,11 +1219,13 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -843,6 +1234,7 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -874,10 +1266,14 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f h1:YORWxaStkWBnWgELOHTmDrqNlFXuVGEbhwbB5iK94bQ= google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -911,17 +1307,28 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -934,6 +1341,11 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -941,6 +1353,23 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= +k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/integration_test.go b/integration_test.go new file mode 100644 index 0000000..ab15cfc --- /dev/null +++ b/integration_test.go @@ -0,0 +1,259 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + "testing" + "time" + + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neofs-api-go/pkg/container" + cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" + "github.com/nspcc-dev/neofs-sdk-go/pkg/policy" + "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +type putResponse struct { + CID string `json:"container_id"` + OID string `json:"object_id"` +} + +func TestIntegration(t *testing.T) { + ctx := context.Background() + aioImage := "nspccdev/neofs-aio-testcontainer:" + versions := []string{"0.24.0", "latest"} + key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") + require.NoError(t, err) + + for _, version := range versions { + aioContainer := createDockerContainer(ctx, t, aioImage+version) + cancel := runServer() + clientPool := getPool(ctx, t, key) + CID := createContainer(ctx, t, clientPool) + + t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) }) + t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID) }) + t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID) }) + + cancel() + err = aioContainer.Terminate(ctx) + require.NoError(t, err) + } +} + +func runServer() context.CancelFunc { + cancelCtx, cancel := context.WithCancel(context.Background()) + + v := getDefaultConfig() + l := newLogger(v) + application := newApp(cancelCtx, WithConfig(v), WithLogger(l)) + go application.Serve(cancelCtx) + + return cancel +} + +func simplePut(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID) { + content := "content of file" + keyAttr, valAttr := "User-Attribute", "user value" + + attributes := map[string]string{ + object.AttributeFileName: "newFile.txt", + keyAttr: valAttr, + } + + var buff bytes.Buffer + w := multipart.NewWriter(&buff) + fw, err := w.CreateFormFile("file", attributes[object.AttributeFileName]) + require.NoError(t, err) + _, err = io.Copy(fw, bytes.NewBufferString(content)) + require.NoError(t, err) + err = w.Close() + require.NoError(t, err) + + request, err := http.NewRequest(http.MethodPost, "http://localhost:8082/upload/"+CID.String(), &buff) + require.NoError(t, err) + request.Header.Set("Content-Type", w.FormDataContentType()) + request.Header.Set("X-Attribute-"+keyAttr, valAttr) + + resp, err := http.DefaultClient.Do(request) + require.NoError(t, err) + defer func() { + err = resp.Body.Close() + require.NoError(t, err) + }() + + addr := &putResponse{} + err = json.NewDecoder(resp.Body).Decode(addr) + require.NoError(t, err) + + err = CID.Parse(addr.CID) + require.NoError(t, err) + + oid := object.NewID() + err = oid.Parse(addr.OID) + require.NoError(t, err) + + objectAddress := object.NewAddress() + objectAddress.SetContainerID(CID) + objectAddress.SetObjectID(oid) + + payload := bytes.NewBuffer(nil) + ops := new(client.GetObjectParams).WithAddress(objectAddress).WithPayloadWriter(payload) + obj, err := clientPool.GetObject(ctx, ops) + require.NoError(t, err) + require.Equal(t, content, payload.String()) + + for _, attribute := range obj.Attributes() { + require.Equal(t, attributes[attribute.Key()], attribute.Value()) + } +} + +func simpleGet(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID) { + content := "content of file" + attributes := map[string]string{ + "some-attr": "some-get-value", + } + + oid := putObject(ctx, t, clientPool, CID, content, attributes) + + resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + oid.String()) + require.NoError(t, err) + defer func() { + err = resp.Body.Close() + require.NoError(t, err) + }() + + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, content, string(data)) + + for k, v := range attributes { + require.Equal(t, v, resp.Header.Get("X-Attribute-"+k)) + } +} + +func getByAttr(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID) { + keyAttr, valAttr := "some-attr", "some-get-by-attr-value" + content := "content of file" + attributes := map[string]string{keyAttr: valAttr} + + oid := putObject(ctx, t, clientPool, CID, content, attributes) + + expectedAttr := map[string]string{ + "X-Attribute-" + keyAttr: valAttr, + "x-object-id": oid.String(), + "x-container-id": CID.String(), + } + + resp, err := http.Get("http://localhost:8082/get_by_attribute/" + CID.String() + "/" + keyAttr + "/" + valAttr) + require.NoError(t, err) + defer func() { + err = resp.Body.Close() + require.NoError(t, err) + }() + + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, content, string(data)) + + for k, v := range expectedAttr { + require.Equal(t, v, resp.Header.Get(k)) + } +} + +func createDockerContainer(ctx context.Context, t *testing.T, image string) testcontainers.Container { + req := testcontainers.ContainerRequest{ + Image: image, + WaitingFor: wait.NewLogStrategy("aio container started").WithStartupTimeout(30 * time.Second), + Name: "aio", + Hostname: "aio", + NetworkMode: "host", + } + aioC, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + require.NoError(t, err) + + return aioC +} + +func getDefaultConfig() *viper.Viper { + v := settings() + v.SetDefault(cfgPeers+".0.address", "127.0.0.1:8080") + v.SetDefault(cfgPeers+".0.weight", 1) + + return v +} + +func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) pool.Pool { + pb := new(pool.Builder) + pb.AddNode("localhost:8080", 1) + + opts := &pool.BuilderOptions{ + Key: &key.PrivateKey, + NodeConnectionTimeout: 5 * time.Second, + NodeRequestTimeout: 5 * time.Second, + } + clientPool, err := pb.Build(ctx, opts) + require.NoError(t, err) + return clientPool +} + +func createContainer(ctx context.Context, t *testing.T, clientPool pool.Pool) *cid.ID { + pp, err := policy.Parse("REP 1") + require.NoError(t, err) + + cnr := container.New( + container.WithPolicy(pp), + container.WithCustomBasicACL(0x0FFFFFFF), + container.WithAttribute(container.AttributeName, "friendlyName"), + container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10))) + cnr.SetOwnerID(clientPool.OwnerID()) + + CID, err := clientPool.PutContainer(ctx, cnr) + require.NoError(t, err) + fmt.Println(CID.String()) + + err = clientPool.WaitForContainerPresence(ctx, CID, &pool.ContainerPollingParams{ + CreationTimeout: 15 * time.Second, + PollInterval: 3 * time.Second, + }) + require.NoError(t, err) + + return CID +} + +func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID, content string, attributes map[string]string) *object.ID { + rawObject := object.NewRaw() + rawObject.SetContainerID(CID) + rawObject.SetOwnerID(clientPool.OwnerID()) + + var attrs []*object.Attribute + for key, val := range attributes { + attr := object.NewAttribute() + attr.SetKey(key) + attr.SetValue(val) + attrs = append(attrs, attr) + } + rawObject.SetAttributes(attrs...) + + ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(bytes.NewBufferString(content)) + oid, err := clientPool.PutObject(ctx, ops) + require.NoError(t, err) + + return oid +} From 0597c0c143e8a91997206aa5b938fb41b86e6b3f Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 8 Sep 2021 09:15:42 +0300 Subject: [PATCH 211/548] [#87] Allow canonical X-Attribute-Neofs-* headers Signed-off-by: Denis Kirillov --- README.md | 3 ++- uploader/filter.go | 22 ++++++++++++---------- uploader/filter_test.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 uploader/filter_test.go diff --git a/README.md b/README.md index 57a1504..d65992a 100644 --- a/README.md +++ b/README.md @@ -357,7 +357,8 @@ You can also add some attributes to your file using the following rules: "X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo: 100500" header to your request the resulting object will get "Ololo: 100500" attribute - * "X-Attribute-NEOFS-*" headers are special, they're used to set internal + * "X-Attribute-NEOFS-*" headers are special + (`-NEOFS-` part can also be `-neofs-` or`-Neofs-`), they're used to set internal NeoFS attributes starting with `__NEOFS__` prefix, for these attributes all dashes get converted to underscores and all letters are capitalized. For example, you can use "X-Attribute-NEOFS-Expiration-Epoch" header to set diff --git a/uploader/filter.go b/uploader/filter.go index 47c789a..0fb853f 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -8,15 +8,15 @@ import ( ) const ( - userAttributeHeaderPrefix = "X-Attribute-" - neofsAttributeHeaderPrefix = "NEOFS-" - - systemAttributePrefix = "__NEOFS__" + userAttributeHeaderPrefix = "X-Attribute-" + systemAttributePrefix = "__NEOFS__" ) -func systemTranslator(key []byte) []byte { - // replace `NEOFS-` with `__NEOFS__` - key = bytes.Replace(key, []byte(neofsAttributeHeaderPrefix), []byte(systemAttributePrefix), 1) +var neofsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")} + +func systemTranslator(key, prefix []byte) []byte { + // replace specified prefix with `__NEOFS__` + key = bytes.Replace(key, prefix, []byte(systemAttributePrefix), 1) // replace `-` with `_` key = bytes.ReplaceAll(key, []byte("-"), []byte("_")) @@ -28,7 +28,6 @@ func systemTranslator(key []byte) []byte { func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]string { result := make(map[string]string) prefix := []byte(userAttributeHeaderPrefix) - system := []byte(neofsAttributeHeaderPrefix) header.VisitAll(func(key, val []byte) { // checks that key and val not empty @@ -45,8 +44,11 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]str key = bytes.TrimPrefix(key, prefix) // checks that it's a system NeoFS header - if bytes.HasPrefix(key, system) { - key = systemTranslator(key) + for _, system := range neofsAttributeHeaderPrefixes { + if bytes.HasPrefix(key, system) { + key = systemTranslator(key, system) + break + } } // checks that attribute key not empty diff --git a/uploader/filter_test.go b/uploader/filter_test.go new file mode 100644 index 0000000..6e04270 --- /dev/null +++ b/uploader/filter_test.go @@ -0,0 +1,32 @@ +package uploader + +import ( + "testing" + + "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" + "github.com/stretchr/testify/require" + "github.com/valyala/fasthttp" +) + +func TestFilter(t *testing.T) { + log, err := logger.New() + require.NoError(t, err) + + req := &fasthttp.RequestHeader{} + req.DisableNormalizing() + req.Set("X-Attribute-Neofs-Expiration-Epoch1", "101") + req.Set("X-Attribute-NEOFS-Expiration-Epoch2", "102") + req.Set("X-Attribute-neofs-Expiration-Epoch3", "103") + req.Set("X-Attribute-MyAttribute", "value") + + expected := map[string]string{ + "__NEOFS__EXPIRATION_EPOCH1": "101", + "MyAttribute": "value", + "__NEOFS__EXPIRATION_EPOCH3": "103", + "__NEOFS__EXPIRATION_EPOCH2": "102", + } + + result := filterHeaders(log, req) + + require.Equal(t, expected, result) +} From 0e2861152d76ddbd76eb8714a6a17858c5f59554 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Fri, 1 Oct 2021 21:03:51 +0300 Subject: [PATCH 212/548] [#91] Replace golint by revive Signed-off-by: Angira Kekteeva --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 79702d9..ebbff9d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -29,7 +29,7 @@ linters: enable: # mandatory linters - govet - - golint + - revive # some default golangci-lint linters - deadcode From 0b364504a7edc6dce1dbc10be2ce7847f5747d57 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 19 Oct 2021 10:52:41 +0300 Subject: [PATCH 213/548] [#92] Support zip download Signed-off-by: Denis Kirillov --- app.go | 2 + downloader/download.go | 134 ++++++++++++++++++++++++++++++++++++++--- go.mod | 8 +-- go.sum | 25 +++++--- integration_test.go | 68 ++++++++++++++++++++- 5 files changed, 209 insertions(+), 28 deletions(-) diff --git a/app.go b/app.go index 52991ff..91c108a 100644 --- a/app.go +++ b/app.go @@ -205,6 +205,8 @@ func (a *app) Serve(ctx context.Context) { r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.DownloadByAttribute)) r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.HeadByAttribute)) a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") + r.GET("/zip/{cid}/{prefix:*}", a.logger(downloader.DownloadZipped)) + a.log.Info("added path /zip/{cid}/{prefix}") // enable metrics if a.cfg.GetBool(cmdMetrics) { a.log.Info("added path /metrics/") diff --git a/downloader/download.go b/downloader/download.go index cd5e87c..543e2f3 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -1,6 +1,7 @@ package downloader import ( + "archive/zip" "context" "errors" "fmt" @@ -308,9 +309,30 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, client. } func (d *Downloader) searchObject(c *fasthttp.RequestCtx, log *zap.Logger, cid *cid.ID, key, val string) (*object.Address, error) { + ids, err := d.searchByAttr(c, cid, key, val) + if err != nil { + return nil, err + } + if len(ids) > 1 { + log.Debug("found multiple objects", + zap.Strings("object_ids", objectIDs(ids).Slice()), + zap.Stringer("show_object_id", ids[0])) + } + + return formAddress(cid, ids[0]), nil +} + +func formAddress(cid *cid.ID, oid *object.ID) *object.Address { + address := object.NewAddress() + address.SetContainerID(cid) + address.SetObjectID(oid) + return address +} + +func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) ([]*object.ID, error) { options := object.NewSearchFilters() options.AddRootFilter() - options.AddFilter(key, val, object.MatchStringEqual) + options.AddFilter(key, val, op) sops := new(client.SearchObjectParams).WithContainerID(cid).WithSearchFilters(options) ids, err := d.pool.SearchObject(c, sops) @@ -320,13 +342,105 @@ func (d *Downloader) searchObject(c *fasthttp.RequestCtx, log *zap.Logger, cid * if len(ids) == 0 { return nil, errObjectNotFound } - if len(ids) > 1 { - log.Debug("found multiple objects", - zap.Strings("object_ids", objectIDs(ids).Slice()), - zap.Stringer("show_object_id", ids[0])) - } - address := object.NewAddress() - address.SetContainerID(cid) - address.SetObjectID(ids[0]) - return address, nil + return ids, nil +} + +func (d *Downloader) searchByPrefix(c *fasthttp.RequestCtx, cid *cid.ID, val string) ([]*object.ID, error) { + return d.search(c, cid, object.AttributeFileName, val, object.MatchCommonPrefix) +} + +func (d *Downloader) searchByAttr(c *fasthttp.RequestCtx, cid *cid.ID, key, val string) ([]*object.ID, error) { + return d.search(c, cid, key, val, object.MatchStringEqual) +} + +// DownloadZipped handles zip by prefix requests. +func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { + status := fasthttp.StatusBadRequest + scid, _ := c.UserValue("cid").(string) + prefix, _ := c.UserValue("prefix").(string) + log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) + + containerID := cid.New() + if err := containerID.Parse(scid); err != nil { + log.Error("wrong container id", zap.Error(err)) + c.Error("wrong container id", status) + return + } + + if err := tokens.StoreBearerToken(c); err != nil { + log.Error("could not fetch and store bearer token", zap.Error(err)) + c.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) + return + } + + ids, err := d.searchByPrefix(c, containerID, prefix) + if err != nil { + log.Error("couldn't find objects", zap.Error(err)) + if errors.Is(err, errObjectNotFound) { + status = fasthttp.StatusNotFound + } + c.Error("couldn't find objects", status) + return + } + + c.Response.Header.Set("Content-Type", "application/zip") + c.Response.Header.Set("Content-Disposition", "attachment; filename=\"archive.zip\"") + c.Response.SetStatusCode(http.StatusOK) + + if err = d.streamFiles(c, containerID, ids); err != nil { + log.Error("couldn't stream files", zap.Error(err)) + c.Error("couldn't stream", fasthttp.StatusInternalServerError) + return + } +} + +func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error { + zipWriter := zip.NewWriter(c) + for _, id := range ids { + var r io.Reader + readerInitCtx, initReader := context.WithCancel(c) + options := new(client.GetObjectParams). + WithAddress(formAddress(cid, id)). + WithPayloadReaderHandler(func(reader io.Reader) { + r = reader + initReader() + }) + + obj, err := d.pool.GetObject(c, options, bearerOpts(c)) + if err != nil { + return err + } + + header := &zip.FileHeader{ + Name: getFilename(obj), + Method: zip.Store, + Modified: time.Now(), + } + entryWriter, err := zipWriter.CreateHeader(header) + if err != nil { + return err + } + + <-readerInitCtx.Done() + _, err = io.Copy(entryWriter, r) + if err != nil { + return err + } + + if err = zipWriter.Flush(); err != nil { + return err + } + } + + return zipWriter.Close() +} + +func getFilename(obj *object.Object) string { + for _, attr := range obj.Attributes() { + if attr.Key() == object.AttributeFileName { + return attr.Value() + } + } + + return "" } diff --git a/go.mod b/go.mod index 4f7f1e7..094d8fd 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/klauspost/compress v1.13.1 // indirect github.com/nspcc-dev/neo-go v0.96.1 - github.com/nspcc-dev/neofs-api-go v1.28.3 + github.com/nspcc-dev/neofs-api-go v1.30.0 github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210728093755-d95d722d6156 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 @@ -27,9 +27,7 @@ require ( github.com/valyala/fasthttp v1.28.0 go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.18.1 - golang.org/x/net v0.0.0-20210716203947-853a461950ff // indirect - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect - google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f // indirect - google.golang.org/grpc v1.39.0 + golang.org/x/tools v0.1.5 // indirect + google.golang.org/grpc v1.41.0 ) diff --git a/go.sum b/go.sum index e588d4b..669c380 100644 --- a/go.sum +++ b/go.sum @@ -165,6 +165,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= @@ -309,6 +310,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -610,8 +612,9 @@ github.com/nspcc-dev/neo-go v0.96.1 h1:JaKWvM/vvQ48bq2ADNj7zH/6Ek38Iqxo22hdu2lhx github.com/nspcc-dev/neo-go v0.96.1/go.mod h1:yguwQBpWMTHx07INKoElJT8Gga1LUdTSi0gT75ywJ68= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-api-go v1.27.1/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= -github.com/nspcc-dev/neofs-api-go v1.28.3 h1:53Ec3hv3LtI3uuG1H8Yp2OKOIdsAqQVfUOyClhnhc9g= github.com/nspcc-dev/neofs-api-go v1.28.3/go.mod h1:YRIzUqBj/lGbmFm8mmCh54ZOzcJKkEIhv2s7ZvSLv3M= +github.com/nspcc-dev/neofs-api-go v1.30.0 h1:Nns7QjmQGk9I5lWMCtmeD9D3de46uyH3H07WBeXTEI0= +github.com/nspcc-dev/neofs-api-go v1.30.0/go.mod h1:KC8T91skIg8juvUh7lQabswQ9J6KmnXErpH8qwDitXA= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= @@ -982,8 +985,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210716203947-853a461950ff h1:j2EK/QoxYNBsXI4R7fQkkRUk8y6wnOBI+6hgPdP/6Ds= -golang.org/x/net v0.0.0-20210716203947-853a461950ff/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b h1:eB48h3HiRycXNy8E0Gf5e0hv7YT6Kt14L/D73G1fuwo= +golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1101,8 +1104,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1115,8 +1118,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1264,8 +1268,8 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f h1:YORWxaStkWBnWgELOHTmDrqNlFXuVGEbhwbB5iK94bQ= -google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210928142010-c7af6a1a74c9 h1:XTH066D35LyHehRwlYhoK3qA+Hcgvg5xREG4kFQEW1Y= +google.golang.org/genproto v0.0.0-20210928142010-c7af6a1a74c9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1290,8 +1294,9 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/integration_test.go b/integration_test.go index ab15cfc..2237e18 100644 --- a/integration_test.go +++ b/integration_test.go @@ -1,6 +1,7 @@ package main import ( + "archive/zip" "bytes" "context" "encoding/json" @@ -8,16 +9,16 @@ import ( "io" "mime/multipart" "net/http" + "sort" "strconv" "testing" "time" - "github.com/nspcc-dev/neofs-api-go/pkg/client" - "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/container" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" + "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-sdk-go/pkg/policy" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/spf13/viper" @@ -47,6 +48,7 @@ func TestIntegration(t *testing.T) { t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) }) t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID) }) t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID) }) + t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID) }) cancel() err = aioContainer.Terminate(ctx) @@ -174,6 +176,66 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid } } +func getZip(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID) { + names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"} + contents := []string{"content of file1", "content of file2"} + attributes1 := map[string]string{object.AttributeFileName: names[0]} + attributes2 := map[string]string{object.AttributeFileName: names[1]} + + putObject(ctx, t, clientPool, CID, contents[0], attributes1) + putObject(ctx, t, clientPool, CID, contents[1], attributes2) + + resp, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder") + require.NoError(t, err) + defer func() { + err = resp.Body.Close() + require.NoError(t, err) + }() + + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + checkZip(t, data, resp.ContentLength, names, contents) + + // check nested folder + resp2, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder/dir") + require.NoError(t, err) + defer func() { + err = resp2.Body.Close() + require.NoError(t, err) + }() + + data2, err := io.ReadAll(resp2.Body) + require.NoError(t, err) + checkZip(t, data2, resp2.ContentLength, names[:1], contents[:1]) +} + +func checkZip(t *testing.T, data []byte, length int64, names, contents []string) { + readerAt := bytes.NewReader(data) + + zipReader, err := zip.NewReader(readerAt, length) + require.NoError(t, err) + + require.Equal(t, len(names), len(zipReader.File)) + + sort.Slice(zipReader.File, func(i, j int) bool { + return zipReader.File[i].FileHeader.Name < zipReader.File[j].FileHeader.Name + }) + + for i, f := range zipReader.File { + require.Equal(t, names[i], f.FileHeader.Name) + + rc, err := f.Open() + require.NoError(t, err) + + all, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, contents[i], string(all)) + + err = rc.Close() + require.NoError(t, err) + } +} + func createDockerContainer(ctx context.Context, t *testing.T, image string) testcontainers.Container { req := testcontainers.ContainerRequest{ Image: image, From 62dd8b1927d559e1aeb99c4c28acf067d38eb033 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 22 Oct 2021 17:19:31 +0300 Subject: [PATCH 214/548] [#92] Use only the latest aio testcontainer Signed-off-by: Denis Kirillov --- integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_test.go b/integration_test.go index 2237e18..7a08da8 100644 --- a/integration_test.go +++ b/integration_test.go @@ -35,7 +35,7 @@ type putResponse struct { func TestIntegration(t *testing.T) { ctx := context.Background() aioImage := "nspccdev/neofs-aio-testcontainer:" - versions := []string{"0.24.0", "latest"} + versions := []string{"latest"} key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) From 9b92c0146c7190b20e87b9c1dcadcb0f24e5cd12 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 29 Oct 2021 16:11:34 +0300 Subject: [PATCH 215/548] [#96] Add zip compression Signed-off-by: Denis Kirillov --- README.md | 13 +++++++++++++ app.go | 2 +- downloader/download.go | 20 +++++++++++++++----- settings.go | 6 ++++++ 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d65992a..37a8866 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,10 @@ if needed. All timing options accept values with suffixes, so "15s" is 15 seconds and "2m" is 2 minutes. +### Zip streaming +The gateway supports downloading files by common prefix (like dir) in zip format. You can enable compression +using config or `HTTP_GW_ZIP_COMPRESSION=true` environment variable. + ### Logging `--verbose` flag enables gRPC logging and there is a number of environment @@ -192,6 +196,9 @@ peers: 0: address: grpc://s01.neofs.devenv:8080 weight: 1 + +zip: + compression: false ``` To know nesting level of variable you need to cut off the prefix `HTTP_GW` from variable and split the rest parts by `_`. @@ -311,6 +318,12 @@ $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m ``` +##### Zip +You can download some dir (files with the same prefix) in zip (it will be compressed if config contains appropriate param): +``` +$ wget http://localhost:8082/zip/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/common/prefix +``` + #### Replies You get object contents in the reply body (if GET method was used), but at the same time you also get a diff --git a/app.go b/app.go index 91c108a..59e0924 100644 --- a/app.go +++ b/app.go @@ -190,7 +190,7 @@ func (a *app) Serve(ctx context.Context) { }() edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) uploader := uploader.New(a.log, a.pool, edts) - downloader, err := downloader.New(ctx, a.log, a.pool) + downloader, err := downloader.New(a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool) if err != nil { a.log.Fatal("failed to create downloader", zap.Error(err)) } diff --git a/downloader/download.go b/downloader/download.go index 543e2f3..1d60805 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -229,14 +229,19 @@ func (o objectIDs) Slice() []string { // Downloader is a download request handler. type Downloader struct { - log *zap.Logger - pool pool.Pool + log *zap.Logger + pool pool.Pool + settings Settings +} + +type Settings struct { + ZipCompression bool } // New creates an instance of Downloader using specified options. -func New(ctx context.Context, log *zap.Logger, conns pool.Pool) (*Downloader, error) { +func New(log *zap.Logger, settings Settings, conns pool.Pool) (*Downloader, error) { var err error - d := &Downloader{log: log, pool: conns} + d := &Downloader{log: log, pool: conns, settings: settings} if err != nil { return nil, fmt.Errorf("failed to get neofs client's reusable artifacts: %w", err) } @@ -396,6 +401,11 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error { zipWriter := zip.NewWriter(c) + compression := zip.Store + if d.settings.ZipCompression { + compression = zip.Deflate + } + for _, id := range ids { var r io.Reader readerInitCtx, initReader := context.WithCancel(c) @@ -413,7 +423,7 @@ func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*obj header := &zip.FileHeader{ Name: getFilename(obj), - Method: zip.Store, + Method: compression, Modified: time.Now(), } entryWriter, err := zipWriter.CreateHeader(header) diff --git a/settings.go b/settings.go index ceffec6..d09b43f 100644 --- a/settings.go +++ b/settings.go @@ -53,6 +53,9 @@ const ( // Peers. cfgPeers = "peers" + // Zip compression. + cfgZipCompression = "zip.compression" + // Application. cfgApplicationName = "app.name" cfgApplicationVersion = "app.version" @@ -134,6 +137,9 @@ func settings() *viper.Viper { // upload header v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) + // zip: + v.SetDefault(cfgZipCompression, false) + if err := v.BindPFlags(flags); err != nil { panic(err) } From 0d2621d34679f49ba0da6c3b5b5b8506463de92a Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 1 Nov 2021 09:54:09 +0300 Subject: [PATCH 216/548] [#94] Fix multiple testcontainer images Signed-off-by: Denis Kirillov --- integration_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/integration_test.go b/integration_test.go index 7a08da8..9ed7837 100644 --- a/integration_test.go +++ b/integration_test.go @@ -33,13 +33,15 @@ type putResponse struct { } func TestIntegration(t *testing.T) { - ctx := context.Background() + rootCtx := context.Background() aioImage := "nspccdev/neofs-aio-testcontainer:" - versions := []string{"latest"} + versions := []string{"0.24.0", "0.25.1", "latest"} key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) for _, version := range versions { + ctx, cancel2 := context.WithCancel(rootCtx) + aioContainer := createDockerContainer(ctx, t, aioImage+version) cancel := runServer() clientPool := getPool(ctx, t, key) @@ -53,6 +55,7 @@ func TestIntegration(t *testing.T) { cancel() err = aioContainer.Terminate(ctx) require.NoError(t, err) + cancel2() } } From 3ef46151d26b00d4681568e4210186580a30bd32 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 2 Nov 2021 18:54:22 +0300 Subject: [PATCH 217/548] [#102] test: Set up pool with SessionExpiration value neofs-node v0.26.1 release tracks session token expiration more closely, so it has to be specified correctly. Signed-off-by: Alex Vanin --- integration_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/integration_test.go b/integration_test.go index 9ed7837..cb9ebd1 100644 --- a/integration_test.go +++ b/integration_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "io" + "math" "mime/multipart" "net/http" "sort" @@ -35,7 +36,7 @@ type putResponse struct { func TestIntegration(t *testing.T) { rootCtx := context.Background() aioImage := "nspccdev/neofs-aio-testcontainer:" - versions := []string{"0.24.0", "0.25.1", "latest"} + versions := []string{"0.24.0", "0.25.1", "0.26.1", "latest"} key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) @@ -269,9 +270,10 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) pool.Pool pb.AddNode("localhost:8080", 1) opts := &pool.BuilderOptions{ - Key: &key.PrivateKey, - NodeConnectionTimeout: 5 * time.Second, - NodeRequestTimeout: 5 * time.Second, + Key: &key.PrivateKey, + NodeConnectionTimeout: 5 * time.Second, + NodeRequestTimeout: 5 * time.Second, + SessionExpirationEpoch: math.MaxUint64, } clientPool, err := pb.Build(ctx, opts) require.NoError(t, err) From 6265fcb26a2868075f37bed4973b411172e7eae8 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 12 Nov 2021 14:37:05 +0300 Subject: [PATCH 218/548] [#105] Add newline to errors Signed-off-by: Denis Kirillov --- app.go | 7 +++++++ downloader/download.go | 21 +++++++++++---------- downloader/head.go | 3 ++- metrics.go | 9 +++++---- pprof.go | 3 ++- response/utils.go | 7 +++++++ uploader/upload.go | 11 ++++++----- 7 files changed, 40 insertions(+), 21 deletions(-) create mode 100644 response/utils.go diff --git a/app.go b/app.go index 59e0924..7dacb18 100644 --- a/app.go +++ b/app.go @@ -13,6 +13,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/nspcc-dev/neofs-http-gw/downloader" + "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/uploader" "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" @@ -197,6 +198,12 @@ func (a *app) Serve(ctx context.Context) { // Configure router. r := router.New() r.RedirectTrailingSlash = true + r.NotFound = func(r *fasthttp.RequestCtx) { + response.Error(r, "Not found", fasthttp.StatusNotFound) + } + r.MethodNotAllowed = func(r *fasthttp.RequestCtx) { + response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) + } r.POST("/upload/{cid}", a.logger(uploader.Upload)) a.log.Info("added path /upload/{cid}") r.GET("/get/{cid}/{oid}", a.logger(downloader.DownloadByAddress)) diff --git a/downloader/download.go b/downloader/download.go index 1d60805..5d50fe9 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -15,6 +15,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/client" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/valyala/fasthttp" @@ -123,7 +124,7 @@ func (r request) receiveFile(clnt client.Object, objectAddress *object.Address) ) if err = tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) - r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) + response.Error(r.RequestCtx, "could not fetch and store bearer token", fasthttp.StatusBadRequest) return } readDetector := newDetector() @@ -177,7 +178,7 @@ func (r request) receiveFile(clnt client.Object, objectAddress *object.Address) if len(contentType) == 0 { if readDetector.err != nil { r.log.Error("could not read object", zap.Error(err)) - r.Error("could not read object", fasthttp.StatusBadRequest) + response.Error(r.RequestCtx, "could not read object", fasthttp.StatusBadRequest) return } readDetector.Wait() @@ -216,7 +217,7 @@ func (r *request) handleNeoFSErr(err error, start time.Time) { msg = errObjectNotFound.Error() } - r.Error(msg, code) + response.Error(r.RequestCtx, msg, code) } func (o objectIDs) Slice() []string { @@ -272,7 +273,7 @@ func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, client.Ob ) if err := address.Parse(val); err != nil { log.Error("wrong object address", zap.Error(err)) - c.Error("wrong object address", fasthttp.StatusBadRequest) + response.Error(c, "wrong object address", fasthttp.StatusBadRequest) return } @@ -296,7 +297,7 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, client. containerID := cid.New() if err := containerID.Parse(scid); err != nil { log.Error("wrong container id", zap.Error(err)) - c.Error("wrong container id", httpStatus) + response.Error(c, "wrong container id", httpStatus) return } @@ -306,7 +307,7 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, client. if errors.Is(err, errObjectNotFound) { httpStatus = fasthttp.StatusNotFound } - c.Error("couldn't search object", httpStatus) + response.Error(c, "couldn't search object", httpStatus) return } @@ -368,13 +369,13 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { containerID := cid.New() if err := containerID.Parse(scid); err != nil { log.Error("wrong container id", zap.Error(err)) - c.Error("wrong container id", status) + response.Error(c, "wrong container id", status) return } if err := tokens.StoreBearerToken(c); err != nil { log.Error("could not fetch and store bearer token", zap.Error(err)) - c.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) + response.Error(c, "could not fetch and store bearer token", fasthttp.StatusBadRequest) return } @@ -384,7 +385,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { if errors.Is(err, errObjectNotFound) { status = fasthttp.StatusNotFound } - c.Error("couldn't find objects", status) + response.Error(c, "couldn't find objects", status) return } @@ -394,7 +395,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { if err = d.streamFiles(c, containerID, ids); err != nil { log.Error("couldn't stream files", zap.Error(err)) - c.Error("couldn't stream", fasthttp.StatusInternalServerError) + response.Error(c, "couldn't stream", fasthttp.StatusInternalServerError) return } } diff --git a/downloader/head.go b/downloader/head.go index 5ecfd19..2a80129 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -7,6 +7,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -18,7 +19,7 @@ func (r request) headObject(clnt client.Object, objectAddress *object.Address) { var start = time.Now() if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) - r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) + response.Error(r.RequestCtx, "could not fetch and store bearer token", fasthttp.StatusBadRequest) return } diff --git a/metrics.go b/metrics.go index a827738..b2f071e 100644 --- a/metrics.go +++ b/metrics.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/fasthttp/router" + "github.com/nspcc-dev/neofs-http-gw/response" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/expfmt" @@ -57,7 +58,7 @@ func metricsHandler(reg prometheus.Gatherer, opts promhttp.HandlerOpts) fasthttp defer func() { <-inFlightSem }() default: - c.Error(fmt.Sprintf( + response.Error(c, fmt.Sprintf( "Limit of concurrent requests reached (%d), try again later.", opts.MaxRequestsInFlight, ), fasthttp.StatusServiceUnavailable) return @@ -76,11 +77,11 @@ func metricsHandler(reg prometheus.Gatherer, opts promhttp.HandlerOpts) fasthttp case promhttp.ContinueOnError: if len(mfs) == 0 { // Still report the error if no metrics have been gathered. - c.Error(err.Error(), fasthttp.StatusServiceUnavailable) + response.Error(c, err.Error(), fasthttp.StatusServiceUnavailable) return } case promhttp.HTTPErrorOnError: - c.Error(err.Error(), fasthttp.StatusServiceUnavailable) + response.Error(c, err.Error(), fasthttp.StatusServiceUnavailable) return } } @@ -106,7 +107,7 @@ func metricsHandler(reg prometheus.Gatherer, opts promhttp.HandlerOpts) fasthttp case promhttp.PanicOnError: panic(err) case promhttp.HTTPErrorOnError: - c.Error(err.Error(), fasthttp.StatusServiceUnavailable) + response.Error(c, err.Error(), fasthttp.StatusServiceUnavailable) return true default: // Do nothing in all other cases, including ContinueOnError. diff --git a/pprof.go b/pprof.go index 0ef1fe5..8831314 100644 --- a/pprof.go +++ b/pprof.go @@ -5,6 +5,7 @@ import ( rtp "runtime/pprof" "github.com/fasthttp/router" + "github.com/nspcc-dev/neofs-http-gw/response" "github.com/valyala/fasthttp" "github.com/valyala/fasthttp/fasthttpadaptor" ) @@ -38,6 +39,6 @@ func pprofHandler() fasthttp.RequestHandler { return } - ctx.Error("Not found", fasthttp.StatusNotFound) + response.Error(ctx, "Not found", fasthttp.StatusNotFound) } } diff --git a/response/utils.go b/response/utils.go new file mode 100644 index 0000000..8a7f383 --- /dev/null +++ b/response/utils.go @@ -0,0 +1,7 @@ +package response + +import "github.com/valyala/fasthttp" + +func Error(r *fasthttp.RequestCtx, msg string, code int) { + r.Error(msg+"\n", code) +} diff --git a/uploader/upload.go b/uploader/upload.go index f43dfb2..1833fa9 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -12,6 +12,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/valyala/fasthttp" @@ -51,12 +52,12 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { ) if err = tokens.StoreBearerToken(c); err != nil { log.Error("could not fetch bearer token", zap.Error(err)) - c.Error("could not fetch bearer token", fasthttp.StatusBadRequest) + response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest) return } if err = cid.Parse(scid); err != nil { log.Error("wrong container id", zap.Error(err)) - c.Error("wrong container id", fasthttp.StatusBadRequest) + response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } defer func() { @@ -75,7 +76,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(u.log, bodyStream, boundary); err != nil { log.Error("could not receive multipart/form", zap.Error(err)) - c.Error("could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) + response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } filtered := filterHeaders(u.log, &c.Request.Header) @@ -112,7 +113,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { if obj, err = u.pool.PutObject(c, ops, client.WithBearer(bt)); err != nil { log.Error("could not store file in neofs", zap.Error(err)) - c.Error("could not store file in neofs", fasthttp.StatusBadRequest) + response.Error(c, "could not store file in neofs", fasthttp.StatusBadRequest) return } @@ -122,7 +123,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { log.Error("could not prepare response", zap.Error(err)) - c.Error("could not prepare response", fasthttp.StatusBadRequest) + response.Error(c, "could not prepare response", fasthttp.StatusBadRequest) return } From 79765fb7cc50d6a780af4021a56633e498ac238f Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 15 Nov 2021 14:12:15 +0300 Subject: [PATCH 219/548] [#106] Update neofs-sdk-go to the latest version Includes: - fix for unavailable endpoints of client pool, - session token cache with request retry in case of missing session token, - migrated neofs client. Signed-off-by: Alex Vanin --- app.go | 4 ++-- downloader/download.go | 20 ++++++++++---------- downloader/head.go | 7 ++++--- go.mod | 3 +-- go.sum | 5 ++--- integration_test.go | 12 ++++++------ main.go | 2 +- tokens/bearer-token.go | 2 +- tokens/bearer-token_test.go | 4 ++-- uploader/filter_test.go | 2 +- uploader/upload.go | 14 +++++++------- 11 files changed, 37 insertions(+), 38 deletions(-) diff --git a/app.go b/app.go index 7dacb18..9b6849d 100644 --- a/app.go +++ b/app.go @@ -15,8 +15,8 @@ import ( "github.com/nspcc-dev/neofs-http-gw/downloader" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/uploader" - "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" - "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" + "github.com/nspcc-dev/neofs-sdk-go/logger" + "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" diff --git a/downloader/download.go b/downloader/download.go index 5d50fe9..13027a3 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -12,12 +12,12 @@ import ( "strings" "time" - "github.com/nspcc-dev/neofs-api-go/pkg/client" - cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" - "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" - "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" + "github.com/nspcc-dev/neofs-sdk-go/client" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -114,7 +114,7 @@ func isValidValue(s string) bool { return true } -func (r request) receiveFile(clnt client.Object, objectAddress *object.Address) { +func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { var ( err error dis = "inline" @@ -189,11 +189,11 @@ func (r request) receiveFile(clnt client.Object, objectAddress *object.Address) r.Response.Header.Set("Content-Disposition", dis+"; filename="+path.Base(filename)) } -func bearerOpts(ctx context.Context) client.CallOption { +func bearerOpts(ctx context.Context) pool.CallOption { if tkn, err := tokens.LoadBearerToken(ctx); err == nil { - return client.WithBearer(tkn) + return pool.WithBearer(tkn) } - return client.WithBearer(nil) + return pool.WithBearer(nil) } func (r *request) handleNeoFSErr(err error, start time.Time) { @@ -263,7 +263,7 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { // byAddress is wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) { +func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, pool.Object, *object.Address)) { var ( address = object.NewAddress() cid, _ = c.UserValue("cid").(string) @@ -286,7 +286,7 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { } // byAttribute is wrapper similar to byAddress. -func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) { +func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Object, *object.Address)) { var ( httpStatus = fasthttp.StatusBadRequest scid, _ = c.UserValue("cid").(string) diff --git a/downloader/head.go b/downloader/head.go index 2a80129..67cc04e 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -5,17 +5,18 @@ import ( "strconv" "time" - "github.com/nspcc-dev/neofs-api-go/pkg/client" - "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" + "github.com/nspcc-dev/neofs-sdk-go/client" + "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" ) const sizeToDetectType = 512 -func (r request) headObject(clnt client.Object, objectAddress *object.Address) { +func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { var start = time.Now() if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) diff --git a/go.mod b/go.mod index 094d8fd..1c5a056 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/klauspost/compress v1.13.1 // indirect github.com/nspcc-dev/neo-go v0.96.1 - github.com/nspcc-dev/neofs-api-go v1.30.0 - github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210728093755-d95d722d6156 + github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211115110427-df6a622c20e8 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 github.com/prometheus/procfs v0.7.1 // indirect diff --git a/go.sum b/go.sum index 669c380..a825e4f 100644 --- a/go.sum +++ b/go.sum @@ -612,15 +612,14 @@ github.com/nspcc-dev/neo-go v0.96.1 h1:JaKWvM/vvQ48bq2ADNj7zH/6Ek38Iqxo22hdu2lhx github.com/nspcc-dev/neo-go v0.96.1/go.mod h1:yguwQBpWMTHx07INKoElJT8Gga1LUdTSi0gT75ywJ68= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-api-go v1.27.1/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= -github.com/nspcc-dev/neofs-api-go v1.28.3/go.mod h1:YRIzUqBj/lGbmFm8mmCh54ZOzcJKkEIhv2s7ZvSLv3M= github.com/nspcc-dev/neofs-api-go v1.30.0 h1:Nns7QjmQGk9I5lWMCtmeD9D3de46uyH3H07WBeXTEI0= github.com/nspcc-dev/neofs-api-go v1.30.0/go.mod h1:KC8T91skIg8juvUh7lQabswQ9J6KmnXErpH8qwDitXA= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210728093755-d95d722d6156 h1:HiCIiXmv9HEbJh6okyW/8OCLNn8r3AbvgmBUtKh+8wE= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210728093755-d95d722d6156/go.mod h1:NOjwgDKeU8ZoXO0k2cwMi5HV3BRJVJLLaXrYpVjF1LU= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211115110427-df6a622c20e8 h1:nhJSZwE2qbrCrVq4TsHlqYlwXePpqD7BsoEsu4TY5vs= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211115110427-df6a622c20e8/go.mod h1:kISVlyRa5l6UIDFigT2AZSW7yUK0QOEmd5mw9WPeYVI= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/integration_test.go b/integration_test.go index cb9ebd1..5bb0733 100644 --- a/integration_test.go +++ b/integration_test.go @@ -16,12 +16,12 @@ import ( "time" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neofs-api-go/pkg/client" - "github.com/nspcc-dev/neofs-api-go/pkg/container" - cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" - "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neofs-sdk-go/pkg/policy" - "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" + "github.com/nspcc-dev/neofs-sdk-go/client" + "github.com/nspcc-dev/neofs-sdk-go/container" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/policy" + "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/spf13/viper" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" diff --git a/main.go b/main.go index 830a26f..966904c 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,7 @@ import ( "os/signal" "syscall" - "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" + "github.com/nspcc-dev/neofs-sdk-go/logger" "github.com/spf13/viper" "go.uber.org/zap" ) diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index c7d16bc..48b109e 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -7,7 +7,7 @@ import ( "errors" "fmt" - "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-sdk-go/token" "github.com/valyala/fasthttp" ) diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index f038014..54258c2 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -4,8 +4,8 @@ import ( "encoding/base64" "testing" - "github.com/nspcc-dev/neofs-api-go/pkg/owner" - "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-sdk-go/owner" + "github.com/nspcc-dev/neofs-sdk-go/token" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" ) diff --git a/uploader/filter_test.go b/uploader/filter_test.go index 6e04270..9cfb435 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -3,7 +3,7 @@ package uploader import ( "testing" - "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" + "github.com/nspcc-dev/neofs-sdk-go/logger" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" ) diff --git a/uploader/upload.go b/uploader/upload.go index 1833fa9..35ef30c 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -7,14 +7,14 @@ import ( "strconv" "time" - "github.com/nspcc-dev/neofs-api-go/pkg/client" - cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" - "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neofs-api-go/pkg/owner" - "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" - "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" + "github.com/nspcc-dev/neofs-sdk-go/client" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/owner" + "github.com/nspcc-dev/neofs-sdk-go/pool" + "github.com/nspcc-dev/neofs-sdk-go/token" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -111,7 +111,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(file) - if obj, err = u.pool.PutObject(c, ops, client.WithBearer(bt)); err != nil { + if obj, err = u.pool.PutObject(c, ops, pool.WithBearer(bt)); err != nil { log.Error("could not store file in neofs", zap.Error(err)) response.Error(c, "could not store file in neofs", fasthttp.StatusBadRequest) return From c55c4a3557d7262f4fe56fc44c9e191144649d75 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 15 Nov 2021 14:12:45 +0300 Subject: [PATCH 220/548] [#106] Update neo-go to the latest version Signed-off-by: Alex Vanin --- go.mod | 2 +- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1c5a056..7a9c769 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/klauspost/compress v1.13.1 // indirect - github.com/nspcc-dev/neo-go v0.96.1 + github.com/nspcc-dev/neo-go v0.97.3 github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211115110427-df6a622c20e8 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 diff --git a/go.sum b/go.sum index a825e4f..db0807a 100644 --- a/go.sum +++ b/go.sum @@ -602,14 +602,17 @@ github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62pr github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E= github.com/nspcc-dev/dbft v0.0.0-20210302103605-cc75991b7cfb/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= +github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02 h1:JgRx27vfGw5WV5QbaNDy0iy2WD1XJO964wwAapaYKLg= +github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= github.com/nspcc-dev/neo-go v0.95.1/go.mod h1:bW07ge1WFXsBgqrcPpLUr6OcyQxHqM26MZNesWMdH0c= github.com/nspcc-dev/neo-go v0.95.3/go.mod h1:t15xRFDVhz5o/pstptdoW9N9JJBNn1hZ6APMNiC6MrY= -github.com/nspcc-dev/neo-go v0.96.1 h1:JaKWvM/vvQ48bq2ADNj7zH/6Ek38Iqxo22hdu2lhxmY= github.com/nspcc-dev/neo-go v0.96.1/go.mod h1:yguwQBpWMTHx07INKoElJT8Gga1LUdTSi0gT75ywJ68= +github.com/nspcc-dev/neo-go v0.97.3 h1:qui/ZYJhga14UwFwrLJj+8t/ms8enTSUwBxKAsjlIes= +github.com/nspcc-dev/neo-go v0.97.3/go.mod h1:31LelE8G5NZwGmePCykqui6BpPyEklTVbOvEh5xEvz8= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-api-go v1.27.1/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= github.com/nspcc-dev/neofs-api-go v1.30.0 h1:Nns7QjmQGk9I5lWMCtmeD9D3de46uyH3H07WBeXTEI0= @@ -816,7 +819,6 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.28.0 h1:ruVmTmZaBR5i67NqnjvvH5gEv0zwHfWtbjoyW98iho4= github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= From 89e03f12808a43ef6bdc346ab0f5ef33e7ff0f98 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 15 Nov 2021 14:44:09 +0300 Subject: [PATCH 221/548] Release v0.17.0 Signed-off-by: Alex Vanin --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1086100..8c5ce44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ This document outlines major changes between releases. +## [0.17.0] - 2021-11-15 + +Support of bulk file download with zip streams and various bug fixes. + +### Fixed +- Allow canonical `X-Attribute-Neofs-*` headers (#87) +- Responses with error message now end with `\n` character (#105) +- Application does not require all neofs endpoints to be healthy at start now + (#103) +- Application now tracks session token errors and recreates tokens in runtime + (#95) + +### Added +- Integration tests with [all-in-one](https://github.com/nspcc-dev/neofs-aio/) + test containers (#85, #94) +- Bulk download support with zip streams (#92, #96) + ## 0.16.1 (28 Jul 2021) New features: @@ -96,3 +113,5 @@ Bugs fixed: Please refer to [Github releases](https://github.com/nspcc-dev/neofs-http-gw/releases/) for older releases. + +[0.17.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.16.1...v0.17.0 From 06be79811173ce142e48d0550a8e553d17227c55 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Mon, 29 Nov 2021 13:35:41 +0300 Subject: [PATCH 222/548] [#109] Add version to logs in start Signed-off-by: Angira Kekteeva --- app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.go b/app.go index 9b6849d..d6c6735 100644 --- a/app.go +++ b/app.go @@ -179,7 +179,7 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*ecds } func (a *app) Wait() { - a.log.Info("starting application") + a.log.Info("starting application", zap.String("version", a.cfg.GetString(cfgApplicationVersion))) <-a.webDone // wait for web-server to be stopped } From cd0633cda03d0401a68c5dd4f892e80228524dfe Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 26 Nov 2021 15:44:07 +0300 Subject: [PATCH 223/548] [#108] Add different expiration header formats Signed-off-by: Denis Kirillov --- README.md | 13 ++++ go.mod | 1 + uploader/filter.go | 65 ++++++++++++++++++ uploader/filter_test.go | 143 ++++++++++++++++++++++++++++++++++++++++ uploader/upload.go | 57 ++++++++++++++++ 5 files changed, 279 insertions(+) diff --git a/README.md b/README.md index 37a8866..697dcf5 100644 --- a/README.md +++ b/README.md @@ -382,6 +382,19 @@ You can also add some attributes to your file using the following rules: HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP option and if request doesn't provide `X-Attribute-Timestamp` header of its own +--- +**NOTE** + +There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority): +1. `X-Attribute-Neofs-Expiration-Epoch: 100` +2. `X-Attribute-Neofs-Expiration-Duration: 24h30m` +3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797` +4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z` + +which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way. + +--- + For successful uploads you get JSON data in reply body with container and object ID, like this: ``` diff --git a/go.mod b/go.mod index 7a9c769..4a4d121 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/klauspost/compress v1.13.1 // indirect github.com/nspcc-dev/neo-go v0.97.3 + github.com/nspcc-dev/neofs-api-go v1.30.0 github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211115110427-df6a622c20e8 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 diff --git a/uploader/filter.go b/uploader/filter.go index 0fb853f..9e4eca5 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -2,7 +2,11 @@ package uploader import ( "bytes" + "fmt" + "strconv" + "time" + "github.com/nspcc-dev/neofs-api-go/v2/object" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -10,6 +14,10 @@ import ( const ( userAttributeHeaderPrefix = "X-Attribute-" systemAttributePrefix = "__NEOFS__" + + expirationDurationAttr = systemAttributePrefix + "EXPIRATION_DURATION" + expirationTimestampAttr = systemAttributePrefix + "EXPIRATION_TIMESTAMP" + expirationRFC3339Attr = systemAttributePrefix + "EXPIRATION_RFC3339" ) var neofsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")} @@ -68,3 +76,60 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]str return result } + +func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations) error { + expirationInEpoch := headers[object.SysAttributeExpEpoch] + + if timeRFC3339, ok := headers[expirationRFC3339Attr]; ok { + expTime, err := time.Parse(time.RFC3339, timeRFC3339) + if err != nil { + return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, expirationRFC3339Attr) + } + + now := time.Now().UTC() + if expTime.Before(now) { + return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, expirationRFC3339Attr) + } + updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) + delete(headers, expirationRFC3339Attr) + } + + if timestamp, ok := headers[expirationTimestampAttr]; ok { + value, err := strconv.ParseInt(timestamp, 10, 64) + if err != nil { + return fmt.Errorf("couldn't parse value %s of header %s", timestamp, expirationTimestampAttr) + } + expTime := time.Unix(value, 0) + + now := time.Now() + if expTime.Before(now) { + return fmt.Errorf("value %s of header %s must be in the future", timestamp, expirationTimestampAttr) + } + updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) + delete(headers, expirationTimestampAttr) + } + + if duration, ok := headers[expirationDurationAttr]; ok { + expDuration, err := time.ParseDuration(duration) + if err != nil { + return fmt.Errorf("couldn't parse value %s of header %s", duration, expirationDurationAttr) + } + if expDuration <= 0 { + return fmt.Errorf("value %s of header %s must be positive", expDuration, expirationDurationAttr) + } + updateExpirationHeader(headers, epochDurations, expDuration) + delete(headers, expirationDurationAttr) + } + + if expirationInEpoch != "" { + headers[object.SysAttributeExpEpoch] = expirationInEpoch + } + + return nil +} + +func updateExpirationHeader(headers map[string]string, durations *epochDurations, expDuration time.Duration) { + epochDuration := durations.msPerBlock * int64(durations.blockPerEpoch) + numEpoch := expDuration.Milliseconds() / epochDuration + headers[object.SysAttributeExpEpoch] = strconv.FormatInt(int64(durations.currentEpoch)+numEpoch, 10) +} diff --git a/uploader/filter_test.go b/uploader/filter_test.go index 9cfb435..a59bcf5 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -1,7 +1,11 @@ package uploader import ( + "strconv" "testing" + "time" + + "github.com/nspcc-dev/neofs-api-go/v2/object" "github.com/nspcc-dev/neofs-sdk-go/logger" "github.com/stretchr/testify/require" @@ -30,3 +34,142 @@ func TestFilter(t *testing.T) { require.Equal(t, expected, result) } + +func TestPrepareExpirationHeader(t *testing.T) { + tomorrow := time.Now().Add(24 * time.Hour) + tomorrowUnix := tomorrow.Unix() + tomorrowUnixNano := tomorrow.UnixNano() + tomorrowUnixMilli := tomorrowUnixNano / 1e6 + + epoch := "100" + duration := "24h" + timestampSec := strconv.FormatInt(tomorrowUnix, 10) + timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10) + timestampNano := strconv.FormatInt(tomorrowUnixNano, 10) + + defaultDurations := &epochDurations{ + currentEpoch: 10, + msPerBlock: 1000, + blockPerEpoch: 101, + } + + epochPerDay := (24 * time.Hour).Milliseconds() / int64(defaultDurations.blockPerEpoch) / defaultDurations.msPerBlock + defaultExpEpoch := strconv.FormatInt(int64(defaultDurations.currentEpoch)+epochPerDay, 10) + + for _, tc := range []struct { + name string + headers map[string]string + durations *epochDurations + err bool + expected map[string]string + }{ + { + name: "valid epoch", + headers: map[string]string{object.SysAttributeExpEpoch: epoch}, + expected: map[string]string{object.SysAttributeExpEpoch: epoch}, + }, + { + name: "valid epoch, valid duration", + headers: map[string]string{ + object.SysAttributeExpEpoch: epoch, + expirationDurationAttr: duration, + }, + durations: defaultDurations, + expected: map[string]string{object.SysAttributeExpEpoch: epoch}, + }, + { + name: "valid epoch, valid rfc3339", + headers: map[string]string{ + object.SysAttributeExpEpoch: epoch, + expirationRFC3339Attr: tomorrow.Format(time.RFC3339), + }, + durations: defaultDurations, + expected: map[string]string{object.SysAttributeExpEpoch: epoch}, + }, + { + name: "valid epoch, valid timestamp sec", + headers: map[string]string{ + object.SysAttributeExpEpoch: epoch, + expirationTimestampAttr: timestampSec, + }, + durations: defaultDurations, + expected: map[string]string{object.SysAttributeExpEpoch: epoch}, + }, + { + name: "valid epoch, valid timestamp milli", + headers: map[string]string{ + object.SysAttributeExpEpoch: epoch, + expirationTimestampAttr: timestampMilli, + }, + durations: defaultDurations, + expected: map[string]string{object.SysAttributeExpEpoch: epoch}, + }, + { + name: "valid epoch, valid timestamp nano", + headers: map[string]string{ + object.SysAttributeExpEpoch: epoch, + expirationTimestampAttr: timestampNano, + }, + durations: defaultDurations, + expected: map[string]string{object.SysAttributeExpEpoch: epoch}, + }, + { + name: "valid timestamp sec", + headers: map[string]string{expirationTimestampAttr: timestampSec}, + durations: defaultDurations, + expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, + }, + { + name: "valid duration", + headers: map[string]string{expirationDurationAttr: duration}, + durations: defaultDurations, + expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, + }, + { + name: "valid rfc3339", + headers: map[string]string{expirationRFC3339Attr: tomorrow.Format(time.RFC3339)}, + durations: defaultDurations, + expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, + }, + { + name: "invalid timestamp sec", + headers: map[string]string{expirationTimestampAttr: "abc"}, + err: true, + }, + { + name: "invalid timestamp sec zero", + headers: map[string]string{expirationTimestampAttr: "0"}, + err: true, + }, + { + name: "invalid duration", + headers: map[string]string{expirationDurationAttr: "1d"}, + err: true, + }, + { + name: "invalid duration negative", + headers: map[string]string{expirationDurationAttr: "-5h"}, + err: true, + }, + { + name: "invalid rfc3339", + headers: map[string]string{expirationRFC3339Attr: "abc"}, + err: true, + }, + { + name: "invalid rfc3339 zero", + headers: map[string]string{expirationRFC3339Attr: time.RFC3339}, + err: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := prepareExpirationHeader(tc.headers, tc.durations) + if tc.err { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.expected, tc.headers) + } + }) + } +} diff --git a/uploader/upload.go b/uploader/upload.go index 35ef30c..4b81d83 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -2,7 +2,9 @@ package uploader import ( "context" + "encoding/binary" "encoding/json" + "fmt" "io" "strconv" "time" @@ -11,6 +13,7 @@ import ( "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-sdk-go/client" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/owner" "github.com/nspcc-dev/neofs-sdk-go/pool" @@ -31,6 +34,12 @@ type Uploader struct { enableDefaultTimestamp bool } +type epochDurations struct { + currentEpoch uint64 + msPerBlock int64 + blockPerEpoch uint64 +} + // New creates a new Uploader using specified logger, connection pool and // other options. func New(log *zap.Logger, conns pool.Pool, enableDefaultTimestamp bool) *Uploader { @@ -80,6 +89,20 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { return } filtered := filterHeaders(u.log, &c.Request.Header) + if needParseExpiration(filtered) { + epochDuration, err := getEpochDurations(c, u.pool) + if err != nil { + log.Error("could not get epoch durations from network info", zap.Error(err)) + response.Error(c, "could parse expiration header, try expiration in epoch", fasthttp.StatusBadRequest) + return + } + if err = prepareExpirationHeader(filtered, epochDuration); err != nil { + log.Error("could not prepare expiration header", zap.Error(err)) + response.Error(c, "could parse expiration header, try expiration in epoch", fasthttp.StatusBadRequest) + return + } + } + attributes := make([]*object.Attribute, 0, len(filtered)) // prepares attributes from filtered headers for key, val := range filtered { @@ -168,3 +191,37 @@ func (pr *putResponse) encode(w io.Writer) error { enc.SetIndent("", "\t") return enc.Encode(pr) } + +func getEpochDurations(ctx context.Context, p pool.Pool) (*epochDurations, error) { + if conn, _, err := p.Connection(); err != nil { + return nil, err + } else if networkInfo, err := conn.NetworkInfo(ctx); err != nil { + return nil, err + } else { + res := &epochDurations{ + currentEpoch: networkInfo.CurrentEpoch(), + msPerBlock: networkInfo.MsPerBlock(), + } + + networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool { + if string(parameter.Key()) == "EpochDuration" { + data := make([]byte, 8) + copy(data, parameter.Value()) + res.blockPerEpoch = binary.LittleEndian.Uint64(data) + return true + } + return false + }) + if res.blockPerEpoch == 0 { + return nil, fmt.Errorf("not found param: EpochDuration") + } + return res, nil + } +} + +func needParseExpiration(headers map[string]string) bool { + _, ok1 := headers[expirationDurationAttr] + _, ok2 := headers[expirationRFC3339Attr] + _, ok3 := headers[expirationTimestampAttr] + return ok1 || ok2 || ok3 +} From 79501077ffeff6d2c17348251bf0c5bbcb3aa457 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 1 Dec 2021 09:26:28 +0300 Subject: [PATCH 224/548] [#100] Update testcontainers version to 0.12.0 Signed-off-by: Denis Kirillov --- go.mod | 3 +-- go.sum | 37 +++++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 4a4d121..93c5fe6 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/fasthttp/router v1.4.1 github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/klauspost/compress v1.13.1 // indirect github.com/nspcc-dev/neo-go v0.97.3 github.com/nspcc-dev/neofs-api-go v1.30.0 @@ -23,7 +22,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 - github.com/testcontainers/testcontainers-go v0.11.1 + github.com/testcontainers/testcontainers-go v0.12.0 github.com/valyala/fasthttp v1.28.0 go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.18.1 diff --git a/go.sum b/go.sum index db0807a..d7baede 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,8 @@ github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1 github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Flaque/filet v0.0.0-20201012163910-45f684403088 h1:PnnQln5IGbhLeJOi6hVs+lCeF+B1dRfFKPGXUAez0Ww= +github.com/Flaque/filet v0.0.0-20201012163910-45f684403088/go.mod h1:TK+jB3mBs+8ZMWhU5BqZKnZWJ1MrLo8etNVg51ueTBo= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= @@ -123,6 +125,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -154,12 +157,14 @@ github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tj github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -183,6 +188,7 @@ github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -284,8 +290,8 @@ github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TT github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= -github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.11+incompatible h1:OqzI/g/W54LczvhnccGqniFoQghHx3pklbLuhfXpqGo= +github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= @@ -321,6 +327,7 @@ github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -528,8 +535,9 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= @@ -573,8 +581,9 @@ github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQ github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM= github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9ObI= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= @@ -661,16 +670,19 @@ github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5X github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc93 h1:x2UMpOOVf3kQ8arv/EsDGwim8PTNqzL1/EYDr/+scOM= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= @@ -745,8 +757,9 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -800,8 +813,8 @@ github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2K github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/testcontainers/testcontainers-go v0.11.1 h1:FiYsB83LSGbiawoV8TpAZGfcCUbtaeeg1SXqEKUxh08= -github.com/testcontainers/testcontainers-go v0.11.1/go.mod h1:/V0UVq+1e7NWYoqTPog179clf0Qp9TOyp4EcXaEFQz8= +github.com/testcontainers/testcontainers-go v0.12.0 h1:SK0NryGHIx7aifF6YqReORL18aGAA4bsDPtikDVCEyg= +github.com/testcontainers/testcontainers-go v0.12.0/go.mod h1:SIndOQXZng0IW8iWU1Js0ynrfZ8xcxrTtDfF6rD2pxs= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -986,8 +999,9 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b h1:eB48h3HiRycXNy8E0Gf5e0hv7YT6Kt14L/D73G1fuwo= golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211108170745-6635138e15ea h1:FosBMXtOc8Tp9Hbo4ltl1WJSrTVewZU8MPnTPY2HdH8= +golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1101,12 +1115,15 @@ golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3 h1:T6tyxxvHMj2L1R2kZg0uNMpS8ZhB9lRa9XRGTCSA65w= +golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From dbbc9e05cfb01ee66dd5ad08466dcaa1eadf3f99 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Tue, 30 Nov 2021 10:22:05 +0300 Subject: [PATCH 225/548] [#111] Move attributes to a separate file Signed-off-by: Angira Kekteeva --- downloader/download.go | 3 ++- downloader/head.go | 3 ++- uploader/filter.go | 38 +++++++++++++++----------------------- uploader/filter_test.go | 38 +++++++++++++++++++------------------- uploader/upload.go | 7 ++++--- utils/attributes.go | 10 ++++++++++ 6 files changed, 52 insertions(+), 47 deletions(-) create mode 100644 utils/attributes.go diff --git a/downloader/download.go b/downloader/download.go index 13027a3..d37ed32 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -14,6 +14,7 @@ import ( "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" + "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/client" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" @@ -152,7 +153,7 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { if !isValidToken(key) || !isValidValue(val) { continue } - r.Response.Header.Set("X-Attribute-"+key, val) + r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) switch key { case object.AttributeFileName: filename = val diff --git a/downloader/head.go b/downloader/head.go index 67cc04e..8e113e6 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -7,6 +7,7 @@ import ( "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" + "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/client" "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/pool" @@ -40,7 +41,7 @@ func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { if !isValidToken(key) || !isValidValue(val) { continue } - r.Response.Header.Set("X-Attribute-"+key, val) + r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) switch key { case object.AttributeTimestamp: value, err := strconv.ParseInt(val, 10, 64) diff --git a/uploader/filter.go b/uploader/filter.go index 9e4eca5..98046c4 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -7,24 +7,16 @@ import ( "time" "github.com/nspcc-dev/neofs-api-go/v2/object" + "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/valyala/fasthttp" "go.uber.org/zap" ) -const ( - userAttributeHeaderPrefix = "X-Attribute-" - systemAttributePrefix = "__NEOFS__" - - expirationDurationAttr = systemAttributePrefix + "EXPIRATION_DURATION" - expirationTimestampAttr = systemAttributePrefix + "EXPIRATION_TIMESTAMP" - expirationRFC3339Attr = systemAttributePrefix + "EXPIRATION_RFC3339" -) - var neofsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")} func systemTranslator(key, prefix []byte) []byte { // replace specified prefix with `__NEOFS__` - key = bytes.Replace(key, prefix, []byte(systemAttributePrefix), 1) + key = bytes.Replace(key, prefix, []byte(utils.SystemAttributePrefix), 1) // replace `-` with `_` key = bytes.ReplaceAll(key, []byte("-"), []byte("_")) @@ -35,7 +27,7 @@ func systemTranslator(key, prefix []byte) []byte { func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]string { result := make(map[string]string) - prefix := []byte(userAttributeHeaderPrefix) + prefix := []byte(utils.UserAttributeHeaderPrefix) header.VisitAll(func(key, val []byte) { // checks that key and val not empty @@ -80,45 +72,45 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]str func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations) error { expirationInEpoch := headers[object.SysAttributeExpEpoch] - if timeRFC3339, ok := headers[expirationRFC3339Attr]; ok { + if timeRFC3339, ok := headers[utils.ExpirationRFC3339Attr]; ok { expTime, err := time.Parse(time.RFC3339, timeRFC3339) if err != nil { - return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, expirationRFC3339Attr) + return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, utils.ExpirationRFC3339Attr) } now := time.Now().UTC() if expTime.Before(now) { - return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, expirationRFC3339Attr) + return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, utils.ExpirationRFC3339Attr) } updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) - delete(headers, expirationRFC3339Attr) + delete(headers, utils.ExpirationRFC3339Attr) } - if timestamp, ok := headers[expirationTimestampAttr]; ok { + if timestamp, ok := headers[utils.ExpirationTimestampAttr]; ok { value, err := strconv.ParseInt(timestamp, 10, 64) if err != nil { - return fmt.Errorf("couldn't parse value %s of header %s", timestamp, expirationTimestampAttr) + return fmt.Errorf("couldn't parse value %s of header %s", timestamp, utils.ExpirationTimestampAttr) } expTime := time.Unix(value, 0) now := time.Now() if expTime.Before(now) { - return fmt.Errorf("value %s of header %s must be in the future", timestamp, expirationTimestampAttr) + return fmt.Errorf("value %s of header %s must be in the future", timestamp, utils.ExpirationTimestampAttr) } updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) - delete(headers, expirationTimestampAttr) + delete(headers, utils.ExpirationTimestampAttr) } - if duration, ok := headers[expirationDurationAttr]; ok { + if duration, ok := headers[utils.ExpirationDurationAttr]; ok { expDuration, err := time.ParseDuration(duration) if err != nil { - return fmt.Errorf("couldn't parse value %s of header %s", duration, expirationDurationAttr) + return fmt.Errorf("couldn't parse value %s of header %s", duration, utils.ExpirationDurationAttr) } if expDuration <= 0 { - return fmt.Errorf("value %s of header %s must be positive", expDuration, expirationDurationAttr) + return fmt.Errorf("value %s of header %s must be positive", expDuration, utils.ExpirationDurationAttr) } updateExpirationHeader(headers, epochDurations, expDuration) - delete(headers, expirationDurationAttr) + delete(headers, utils.ExpirationDurationAttr) } if expirationInEpoch != "" { diff --git a/uploader/filter_test.go b/uploader/filter_test.go index a59bcf5..4dc8e0c 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -6,7 +6,7 @@ import ( "time" "github.com/nspcc-dev/neofs-api-go/v2/object" - + "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/logger" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" @@ -71,8 +71,8 @@ func TestPrepareExpirationHeader(t *testing.T) { { name: "valid epoch, valid duration", headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - expirationDurationAttr: duration, + object.SysAttributeExpEpoch: epoch, + utils.ExpirationDurationAttr: duration, }, durations: defaultDurations, expected: map[string]string{object.SysAttributeExpEpoch: epoch}, @@ -81,7 +81,7 @@ func TestPrepareExpirationHeader(t *testing.T) { name: "valid epoch, valid rfc3339", headers: map[string]string{ object.SysAttributeExpEpoch: epoch, - expirationRFC3339Attr: tomorrow.Format(time.RFC3339), + utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339), }, durations: defaultDurations, expected: map[string]string{object.SysAttributeExpEpoch: epoch}, @@ -89,8 +89,8 @@ func TestPrepareExpirationHeader(t *testing.T) { { name: "valid epoch, valid timestamp sec", headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - expirationTimestampAttr: timestampSec, + object.SysAttributeExpEpoch: epoch, + utils.ExpirationTimestampAttr: timestampSec, }, durations: defaultDurations, expected: map[string]string{object.SysAttributeExpEpoch: epoch}, @@ -98,8 +98,8 @@ func TestPrepareExpirationHeader(t *testing.T) { { name: "valid epoch, valid timestamp milli", headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - expirationTimestampAttr: timestampMilli, + object.SysAttributeExpEpoch: epoch, + utils.ExpirationTimestampAttr: timestampMilli, }, durations: defaultDurations, expected: map[string]string{object.SysAttributeExpEpoch: epoch}, @@ -107,58 +107,58 @@ func TestPrepareExpirationHeader(t *testing.T) { { name: "valid epoch, valid timestamp nano", headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - expirationTimestampAttr: timestampNano, + object.SysAttributeExpEpoch: epoch, + utils.ExpirationTimestampAttr: timestampNano, }, durations: defaultDurations, expected: map[string]string{object.SysAttributeExpEpoch: epoch}, }, { name: "valid timestamp sec", - headers: map[string]string{expirationTimestampAttr: timestampSec}, + headers: map[string]string{utils.ExpirationTimestampAttr: timestampSec}, durations: defaultDurations, expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, }, { name: "valid duration", - headers: map[string]string{expirationDurationAttr: duration}, + headers: map[string]string{utils.ExpirationDurationAttr: duration}, durations: defaultDurations, expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, }, { name: "valid rfc3339", - headers: map[string]string{expirationRFC3339Attr: tomorrow.Format(time.RFC3339)}, + headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)}, durations: defaultDurations, expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, }, { name: "invalid timestamp sec", - headers: map[string]string{expirationTimestampAttr: "abc"}, + headers: map[string]string{utils.ExpirationTimestampAttr: "abc"}, err: true, }, { name: "invalid timestamp sec zero", - headers: map[string]string{expirationTimestampAttr: "0"}, + headers: map[string]string{utils.ExpirationTimestampAttr: "0"}, err: true, }, { name: "invalid duration", - headers: map[string]string{expirationDurationAttr: "1d"}, + headers: map[string]string{utils.ExpirationDurationAttr: "1d"}, err: true, }, { name: "invalid duration negative", - headers: map[string]string{expirationDurationAttr: "-5h"}, + headers: map[string]string{utils.ExpirationDurationAttr: "-5h"}, err: true, }, { name: "invalid rfc3339", - headers: map[string]string{expirationRFC3339Attr: "abc"}, + headers: map[string]string{utils.ExpirationRFC3339Attr: "abc"}, err: true, }, { name: "invalid rfc3339 zero", - headers: map[string]string{expirationRFC3339Attr: time.RFC3339}, + headers: map[string]string{utils.ExpirationRFC3339Attr: time.RFC3339}, err: true, }, } { diff --git a/uploader/upload.go b/uploader/upload.go index 4b81d83..c2d06c4 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -11,6 +11,7 @@ import ( "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" + "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/client" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/netmap" @@ -220,8 +221,8 @@ func getEpochDurations(ctx context.Context, p pool.Pool) (*epochDurations, error } func needParseExpiration(headers map[string]string) bool { - _, ok1 := headers[expirationDurationAttr] - _, ok2 := headers[expirationRFC3339Attr] - _, ok3 := headers[expirationTimestampAttr] + _, ok1 := headers[utils.ExpirationDurationAttr] + _, ok2 := headers[utils.ExpirationRFC3339Attr] + _, ok3 := headers[utils.ExpirationTimestampAttr] return ok1 || ok2 || ok3 } diff --git a/utils/attributes.go b/utils/attributes.go new file mode 100644 index 0000000..814d7b1 --- /dev/null +++ b/utils/attributes.go @@ -0,0 +1,10 @@ +package utils + +const ( + UserAttributeHeaderPrefix = "X-Attribute-" + SystemAttributePrefix = "__NEOFS__" + + ExpirationDurationAttr = SystemAttributePrefix + "EXPIRATION_DURATION" + ExpirationTimestampAttr = SystemAttributePrefix + "EXPIRATION_TIMESTAMP" + ExpirationRFC3339Attr = SystemAttributePrefix + "EXPIRATION_RFC3339" +) From d6dd244756e59d6e3ef1df8cae97fd6e4c79df50 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Tue, 30 Nov 2021 12:25:50 +0300 Subject: [PATCH 226/548] [#111] Fix expiration epoch headers Signed-off-by: Angira Kekteeva --- downloader/download.go | 23 +++++++++++++++++++++++ downloader/download_test.go | 23 +++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 downloader/download_test.go diff --git a/downloader/download.go b/downloader/download.go index d37ed32..7cd25e5 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -153,6 +153,9 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { if !isValidToken(key) || !isValidValue(val) { continue } + if strings.HasPrefix(key, utils.SystemAttributePrefix) { + key = systemBackwardTranslator(key) + } r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) switch key { case object.AttributeFileName: @@ -190,6 +193,26 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { r.Response.Header.Set("Content-Disposition", dis+"; filename="+path.Base(filename)) } +// systemBackwardTranslator is used to convert headers looking like '__NEOFS__ATTR_NAME' to 'Neofs-Attr-Name'. +func systemBackwardTranslator(key string) string { + // trim specified prefix '__NEOFS__' + key = strings.TrimPrefix(key, utils.SystemAttributePrefix) + + var res strings.Builder + res.WriteString("Neofs-") + + strs := strings.Split(key, "_") + for i, s := range strs { + s = strings.Title(strings.ToLower(s)) + res.WriteString(s) + if i != len(strs)-1 { + res.WriteString("-") + } + } + + return res.String() +} + func bearerOpts(ctx context.Context) pool.CallOption { if tkn, err := tokens.LoadBearerToken(ctx); err == nil { return pool.WithBearer(tkn) diff --git a/downloader/download_test.go b/downloader/download_test.go new file mode 100644 index 0000000..a47d12a --- /dev/null +++ b/downloader/download_test.go @@ -0,0 +1,23 @@ +package downloader + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSystemBackwardTranslator(t *testing.T) { + input := []string{ + "__NEOFS__EXPIRATION_EPOCH", + "__NEOFS__RANDOM_ATTR", + } + expected := []string{ + "Neofs-Expiration-Epoch", + "Neofs-Random-Attr", + } + + for i, str := range input { + res := systemBackwardTranslator(str) + require.Equal(t, expected[i], res) + } +} From 1bb6c5f384aaad4661e6dd94ddc5c71e1dd57146 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Mon, 29 Nov 2021 22:11:07 +0300 Subject: [PATCH 227/548] [#111] Fix X- headers Signed-off-by: Angira Kekteeva --- downloader/download.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 7cd25e5..d669cfb 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -175,9 +175,9 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { contentType = val } } - r.Response.Header.Set("x-object-id", obj.ID().String()) - r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) - r.Response.Header.Set("x-container-id", obj.ContainerID().String()) + r.Response.Header.Set("X-Object-Id", obj.ID().String()) + r.Response.Header.Set("X-Owner-Id", obj.OwnerID().String()) + r.Response.Header.Set("X-Container-Id", obj.ContainerID().String()) if len(contentType) == 0 { if readDetector.err != nil { From 71e846a4fca1df8b92344abf1f8bd7cbc3a06273 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 7 Dec 2021 14:57:16 +0300 Subject: [PATCH 228/548] [#115] Update sdk version to support priority Signed-off-by: Denis Kirillov --- README.md | 19 +++++---- app.go | 9 +++- go.mod | 12 ++---- go.sum | 101 +++++++++----------------------------------- integration_test.go | 3 +- settings.go | 1 + uploader/upload.go | 7 ++- 7 files changed, 50 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index 697dcf5..2f0858f 100644 --- a/README.md +++ b/README.md @@ -69,23 +69,24 @@ In general, everything available as CLI parameter can also be specified via environment variables, so they're not specifically mentioned in most cases (see `--help` also). If you prefer a config file you can use it in yaml format. -### Nodes and weights +### Nodes: weights and priorities You can specify multiple `-p` options to add more NeoFS nodes, this will make -gateway spread requests equally among them (using weight 1 for every node): +gateway spread requests equally among them (using weight 1 and priority 1 for every node): ``` $ neofs-http-gw -p 192.168.130.72:8080 -p 192.168.130.71:8080 ``` -If you want some specific load distribution proportions, use weights, but they -can only be specified via environment variables: +If you want some specific load distribution proportions, use weights and priorities: ``` -$ HTTP_GW_PEERS_0_ADDRESS=192.168.130.72:8080 HTTP_GW_PEERS_0_WEIGHT=9 \ - HTTP_GW_PEERS_1_ADDRESS=192.168.130.71:8080 HTTP_GW_PEERS_1_WEIGHT=1 neofs-http-gw +$ HTTP_GW_PEERS_0_ADDRESS=192.168.130.71:8080 HTTP_GW_PEERS_0_WEIGHT=1 HTTP_GW_PEERS_0_PRIORITY=1 \ + HTTP_GW_PEERS_1_ADDRESS=192.168.130.72:8080 HTTP_GW_PEERS_1_WEIGHT=9 HTTP_GW_PEERS_1_PRIORITY=2 \ + HTTP_GW_PEERS_2_ADDRESS=192.168.130.73:8080 HTTP_GW_PEERS_2_WEIGHT=1 HTTP_GW_PEERS_2_PRIORITY=2 \ + neofs-http-gw ``` -This command will make gateway use 192.168.130.72 for 90% of requests and -192.168.130.71 for remaining 10%. +This command will make gateway use 192.168.130.71 while it is healthy. Otherwise, it will make the gateway use +192.168.130.72 for 90% of requests and 192.168.130.73 for remaining 10%. ### Keys You can provide wallet via `--wallet` or `-w` flag also you can specify account address using `--address` @@ -196,6 +197,7 @@ peers: 0: address: grpc://s01.neofs.devenv:8080 weight: 1 + priority: 1 zip: compression: false @@ -207,6 +209,7 @@ For example variable `HTTP_GW_PEERS_0_WEIGHT=1` will be transformed to: peers: 0: weight: 1 + priority: 1 ``` If parameter doesn't support environment variable (e.g. `--listen_address 0.0.0.0:8082`) form it is used as is: diff --git a/app.go b/app.go index d6c6735..267c92c 100644 --- a/app.go +++ b/app.go @@ -103,14 +103,19 @@ func newApp(ctx context.Context, opt ...Option) App { for i := 0; ; i++ { address := a.cfg.GetString(cfgPeers + "." + strconv.Itoa(i) + ".address") weight := a.cfg.GetFloat64(cfgPeers + "." + strconv.Itoa(i) + ".weight") + priority := a.cfg.GetInt(cfgPeers + "." + strconv.Itoa(i) + ".priority") if address == "" { break } if weight <= 0 { // unspecified or wrong weight = 1 } - pb.AddNode(address, weight) - a.log.Info("add connection", zap.String("address", address), zap.Float64("weight", weight)) + if priority <= 0 { // unspecified or wrong + priority = 1 + } + pb.AddNode(address, priority, weight) + a.log.Info("add connection", zap.String("address", address), + zap.Float64("weight", weight), zap.Int("priority", priority)) } opts := &pool.BuilderOptions{ Key: key, diff --git a/go.mod b/go.mod index 93c5fe6..256f6b9 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,15 @@ module github.com/nspcc-dev/neofs-http-gw go 1.16 require ( - github.com/DataDog/zstd v1.4.8 // indirect github.com/andybalholm/brotli v1.0.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect - github.com/dgraph-io/badger/v2 v2.2007.3 // indirect - github.com/dgraph-io/ristretto v0.1.0 // indirect - github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/fasthttp/router v1.4.1 - github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect + github.com/golang/mock v1.6.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/klauspost/compress v1.13.1 // indirect - github.com/nspcc-dev/neo-go v0.97.3 - github.com/nspcc-dev/neofs-api-go v1.30.0 - github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211115110427-df6a622c20e8 + github.com/nspcc-dev/neo-go v0.98.0 + github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1 + github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 github.com/prometheus/procfs v0.7.1 // indirect diff --git a/go.sum b/go.sum index d7baede..044e645 100644 --- a/go.sum +++ b/go.sum @@ -57,9 +57,6 @@ github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1 github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= -github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= -github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Flaque/filet v0.0.0-20201012163910-45f684403088 h1:PnnQln5IGbhLeJOi6hVs+lCeF+B1dRfFKPGXUAez0Ww= github.com/Flaque/filet v0.0.0-20201012163910-45f684403088/go.mod h1:TK+jB3mBs+8ZMWhU5BqZKnZWJ1MrLo8etNVg51ueTBo= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= @@ -80,13 +77,10 @@ github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+V github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= -github.com/Workiva/go-datastructures v1.0.53/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/ishell/v2 v2.0.2/go.mod h1:E4oTCXfo6QjoCart0QYa5m9w4S+deXs/P/9jA77A9Bs= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= @@ -98,12 +92,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= -github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= -github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= -github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= -github.com/alicebob/miniredis/v2 v2.15.1 h1:Fw+ixAJPmKhCLBqDwHlTDqxUxp0xjEwXczEpt1B6r7k= -github.com/alicebob/miniredis/v2 v2.15.1/go.mod h1:gquAfGbzn92jvtrSC69+6zZnwSODVXVpYDRaGhWaL6I= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= @@ -111,7 +100,6 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1 h1:zFRi26YWd7NIorBXe8UkevRl0dIvk/AnXHWaAiZG+Yk= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -151,8 +139,6 @@ github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3k github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -169,7 +155,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= @@ -239,8 +224,6 @@ github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -253,13 +236,12 @@ github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= @@ -273,18 +255,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= -github.com/dgraph-io/badger/v2 v2.2007.3 h1:Sl9tQWz92WCbVSe8pj04Tkqlm2boW+KAxd+XSs58SQI= -github.com/dgraph-io/badger/v2 v2.2007.3/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= -github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -304,7 +276,6 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNE github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -315,7 +286,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= @@ -326,8 +296,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -376,8 +346,6 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= -github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -429,8 +397,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -494,6 +463,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -536,13 +506,14 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -569,7 +540,6 @@ github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WT github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= @@ -608,30 +578,22 @@ github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+d github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= -github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E= -github.com/nspcc-dev/dbft v0.0.0-20210302103605-cc75991b7cfb/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02 h1:JgRx27vfGw5WV5QbaNDy0iy2WD1XJO964wwAapaYKLg= github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= -github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= -github.com/nspcc-dev/neo-go v0.95.1/go.mod h1:bW07ge1WFXsBgqrcPpLUr6OcyQxHqM26MZNesWMdH0c= -github.com/nspcc-dev/neo-go v0.95.3/go.mod h1:t15xRFDVhz5o/pstptdoW9N9JJBNn1hZ6APMNiC6MrY= -github.com/nspcc-dev/neo-go v0.96.1/go.mod h1:yguwQBpWMTHx07INKoElJT8Gga1LUdTSi0gT75ywJ68= -github.com/nspcc-dev/neo-go v0.97.3 h1:qui/ZYJhga14UwFwrLJj+8t/ms8enTSUwBxKAsjlIes= -github.com/nspcc-dev/neo-go v0.97.3/go.mod h1:31LelE8G5NZwGmePCykqui6BpPyEklTVbOvEh5xEvz8= -github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= -github.com/nspcc-dev/neofs-api-go v1.27.1/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= -github.com/nspcc-dev/neofs-api-go v1.30.0 h1:Nns7QjmQGk9I5lWMCtmeD9D3de46uyH3H07WBeXTEI0= -github.com/nspcc-dev/neofs-api-go v1.30.0/go.mod h1:KC8T91skIg8juvUh7lQabswQ9J6KmnXErpH8qwDitXA= +github.com/nspcc-dev/neo-go v0.98.0 h1:yyW4sgY88/pLf0949qmgfkQXzRKC3CI/WyhqXNnwMd8= +github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= +github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1 h1:CGA56mhLLduWRuMHcWujP5Ek+gAnXHk0WuIWkG65G1s= +github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211115110427-df6a622c20e8 h1:nhJSZwE2qbrCrVq4TsHlqYlwXePpqD7BsoEsu4TY5vs= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211115110427-df6a622c20e8/go.mod h1:kISVlyRa5l6UIDFigT2AZSW7yUK0QOEmd5mw9WPeYVI= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477 h1:JC+jt4ARpMV/L3OqPHBKxAmbMabU7RYl/L4KgBz3yPs= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= @@ -684,12 +646,9 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -738,8 +697,8 @@ github.com/prometheus/procfs v0.7.1/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -766,20 +725,15 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -788,7 +742,6 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= @@ -815,12 +768,9 @@ github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKk github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/testcontainers/testcontainers-go v0.12.0 h1:SK0NryGHIx7aifF6YqReORL18aGAA4bsDPtikDVCEyg= github.com/testcontainers/testcontainers-go v0.12.0/go.mod h1:SIndOQXZng0IW8iWU1Js0ynrfZ8xcxrTtDfF6rD2pxs= -github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -843,7 +793,6 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -851,13 +800,10 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= -github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da h1:NimzV1aGyq29m5ukMK0AMWEhFaL/lrEOaephfuoiARg= -github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= @@ -895,7 +841,6 @@ golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -999,7 +944,6 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211108170745-6635138e15ea h1:FosBMXtOc8Tp9Hbo4ltl1WJSrTVewZU8MPnTPY2HdH8= golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1034,7 +978,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1050,7 +993,6 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1120,12 +1062,10 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3 h1:T6tyxxvHMj2L1R2kZg0uNMpS8ZhB9lRa9XRGTCSA65w= golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1136,9 +1076,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1285,9 +1224,8 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210928142010-c7af6a1a74c9 h1:XTH066D35LyHehRwlYhoK3qA+Hcgvg5xREG4kFQEW1Y= -google.golang.org/genproto v0.0.0-20210928142010-c7af6a1a74c9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1312,7 +1250,6 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/integration_test.go b/integration_test.go index 5bb0733..d38655d 100644 --- a/integration_test.go +++ b/integration_test.go @@ -261,13 +261,14 @@ func getDefaultConfig() *viper.Viper { v := settings() v.SetDefault(cfgPeers+".0.address", "127.0.0.1:8080") v.SetDefault(cfgPeers+".0.weight", 1) + v.SetDefault(cfgPeers+".0.priority", 1) return v } func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) pool.Pool { pb := new(pool.Builder) - pb.AddNode("localhost:8080", 1) + pb.AddNode("localhost:8080", 1, 1) opts := &pool.BuilderOptions{ Key: &key.PrivateKey, diff --git a/settings.go b/settings.go index d09b43f..5ab6323 100644 --- a/settings.go +++ b/settings.go @@ -193,6 +193,7 @@ func settings() *viper.Viper { for i := range *peers { v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".address", (*peers)[i]) v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".weight", 1) + v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".priority", 1) } } diff --git a/uploader/upload.go b/uploader/upload.go index c2d06c4..ef91b97 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -9,6 +9,8 @@ import ( "strconv" "time" + apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" + "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" @@ -196,9 +198,12 @@ func (pr *putResponse) encode(w io.Writer) error { func getEpochDurations(ctx context.Context, p pool.Pool) (*epochDurations, error) { if conn, _, err := p.Connection(); err != nil { return nil, err - } else if networkInfo, err := conn.NetworkInfo(ctx); err != nil { + } else if networkInfoRes, err := conn.NetworkInfo(ctx); err != nil { + return nil, err + } else if err = apistatus.ErrFromStatus(networkInfoRes.Status()); err != nil { return nil, err } else { + networkInfo := networkInfoRes.Info() res := &epochDurations{ currentEpoch: networkInfo.CurrentEpoch(), msPerBlock: networkInfo.MsPerBlock(), From 567f54d82d830eefdfbac3e03ebf96919840a5a1 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 9 Dec 2021 17:54:43 +0300 Subject: [PATCH 229/548] [#115] Fix tests Signed-off-by: Denis Kirillov --- integration_test.go | 2 +- uploader/upload.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/integration_test.go b/integration_test.go index d38655d..ffcd92f 100644 --- a/integration_test.go +++ b/integration_test.go @@ -36,7 +36,7 @@ type putResponse struct { func TestIntegration(t *testing.T) { rootCtx := context.Background() aioImage := "nspccdev/neofs-aio-testcontainer:" - versions := []string{"0.24.0", "0.25.1", "0.26.1", "latest"} + versions := []string{"0.24.0", "0.25.1", "0.26.1", "0.27.0", "latest"} key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) diff --git a/uploader/upload.go b/uploader/upload.go index ef91b97..1861516 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -9,12 +9,11 @@ import ( "strconv" "time" - apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" - "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/client" + apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" From 2110e924e907ec84e56133f14ad18560f7f2b16f Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Fri, 10 Dec 2021 15:08:59 +0300 Subject: [PATCH 230/548] Release v0.18.0 Signed-off-by: Angira Kekteeva --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c5ce44..bb55c5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ This document outlines major changes between releases. +## [0.18.0] - 2021-12-10 + +### Fixed +- System headers format (#111) + +### Added +- Different formats to set object's expiration: in epoch, duration, timestamp, + RFC3339 (#108) +- Support of nodes priority (#115) + +### Changed +- Updated testcontainers dependency (#100) + ## [0.17.0] - 2021-11-15 Support of bulk file download with zip streams and various bug fixes. @@ -115,3 +128,4 @@ releases](https://github.com/nspcc-dev/neofs-http-gw/releases/) for older releases. [0.17.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.16.1...v0.17.0 +[0.18.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.17.0...v0.18.0 \ No newline at end of file From c21217bc62aca9fe759a69164d53188c0b98514c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 27 Dec 2021 15:19:28 +0300 Subject: [PATCH 231/548] [#120] Fix project name Signed-off-by: Denis Kirillov --- CONTRIBUTING.md | 2 +- README.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 130a6e3..e73dbf8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,7 +28,7 @@ send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: ### Set up your GitHub Repository -Fork [NeoFS HTTP Protocol Gateway +Fork [NeoFS HTTP Gateway upstream](https://github.com/nspcc-dev/neofs-http-gw/fork) source repository to your own personal repository. Copy the URL of your fork (you will need it for the `git clone` command below). diff --git a/README.md b/README.md index 2f0858f..01bc002 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ ![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/nspcc-dev/neofs-http-gw?sort=semver) ![License](https://img.shields.io/github/license/nspcc-dev/neofs-http-gw.svg?style=popout) -# NeoFS HTTP Protocol Gateway +# NeoFS HTTP Gateway -NeoFS HTTP Protocol Gateway bridges NeoFS internal protocol and HTTP standard. +NeoFS HTTP Gateway bridges NeoFS internal protocol and HTTP standard. - you can download one file per request from NeoFS Network - you can upload one file per request into the NeoFS Network @@ -411,7 +411,7 @@ object ID, like this: You can always upload files to public containers (open for anyone to put objects into), but for restricted containers you need to explicitly allow PUT -operations for request signed with your HTTP Protocol Gateway keys. +operations for request signed with your HTTP Gateway keys. If your don't want to manage gateway's secret keys and adjust eACL rules when gateway configuration changes (new gate, key rotation, etc) or you plan to use @@ -428,7 +428,7 @@ documentation for more details). There are two options to pass them to gateway: For example you have a mobile application frontend with a backend part storing data in NeoFS. When user authorizes in mobile app, the backend issues a NeoFS Bearer token and provides it to the frontend. Then the mobile app may generate -some data and upload it via any available NeoFS HTTP Protocol Gateway by adding +some data and upload it via any available NeoFS HTTP Gateway by adding the corresponding header to the upload request. Accessing the ACL protected data works the same way. From c11b2332f9c13d75c4224151e803e40ac3515278 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Mon, 24 Jan 2022 17:04:34 +0300 Subject: [PATCH 232/548] [#123] Add url-encoded queries for attributes Signed-off-by: Angira Kekteeva --- README.md | 13 +++++++++++++ downloader/download.go | 5 +++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 01bc002..fd87190 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,9 @@ where `$ATTRIBUTE_NAME` is the name of the attribute we want to use, `$ATTRIBUTE_VALUE` is the value of this attribute that the target object should have. +**NB!** The attribute key and value must be url encoded, i.e., if you want to download an object with the attribute value +`a cat`, the value in the request must be `a+cat`. In the same way with the attribute key. + If multiple objects have specified attribute with specified value, then the first one of them is returned (and you can't get others via this interface). @@ -306,6 +309,11 @@ Example for file name attribute: ``` $ wget http://localhost:8082/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/FileName/cat.jpeg ``` +Or when the filename includes special symbols: +``` +$ wget http://localhost:8082/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/FileName/cat+jpeg # means 'cat jpeg' +$ wget http://localhost:8082/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/FileName/cat%25jpeg # means 'cat%jpeg' +``` Some other user-defined attribute: @@ -313,6 +321,11 @@ Some other user-defined attribute: $ wget http://localhost:8082/get_by_attribute/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/Ololo/100500 ``` +Or when the attribute includes special symbols: +``` +$ wget http://localhost:8082/get_by_attribute/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/Olo%2Blo/100500 # means Olo+lo +``` + An optional `download=true` argument for `Content-Disposition` management is also supported (more on that below): diff --git a/downloader/download.go b/downloader/download.go index d669cfb..e71873e 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "net/url" "path" "strconv" "strings" @@ -314,8 +315,8 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Ob var ( httpStatus = fasthttp.StatusBadRequest scid, _ = c.UserValue("cid").(string) - key, _ = c.UserValue("attr_key").(string) - val, _ = c.UserValue("attr_val").(string) + key, _ = url.QueryUnescape(c.UserValue("attr_key").(string)) + val, _ = url.QueryUnescape(c.UserValue("attr_val").(string)) log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ) containerID := cid.New() From 9d085740e0bf2448872a7370b0c04e0f338fd8a6 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Wed, 26 Jan 2022 17:54:10 +0300 Subject: [PATCH 233/548] [#123] Add url-encoded queries for prefix values Signed-off-by: Angira Kekteeva --- downloader/download.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/downloader/download.go b/downloader/download.go index e71873e..93aa020 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -388,7 +388,7 @@ func (d *Downloader) searchByAttr(c *fasthttp.RequestCtx, cid *cid.ID, key, val func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { status := fasthttp.StatusBadRequest scid, _ := c.UserValue("cid").(string) - prefix, _ := c.UserValue("prefix").(string) + prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string)) log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) containerID := cid.New() From 2764fabf045b4d3b940c6095c89d3282da186e24 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 15 Feb 2022 11:27:51 +0300 Subject: [PATCH 234/548] [#125] Use the same HEAD, GET headers formation Signed-off-by: Denis Kirillov --- downloader/download.go | 16 ++++++++-------- downloader/head.go | 16 +++++++++++----- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 93aa020..03b2729 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -146,7 +146,7 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { dis = "attachment" } r.Response.SetBodyStream(readDetector.MultiReader(), int(obj.PayloadSize())) - r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) + r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10)) var contentType string for _, attr := range obj.Attributes() { key := attr.Key() @@ -170,15 +170,15 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { zap.Error(err)) continue } - r.Response.Header.Set("Last-Modified", + r.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat)) case object.AttributeContentType: contentType = val } } - r.Response.Header.Set("X-Object-Id", obj.ID().String()) - r.Response.Header.Set("X-Owner-Id", obj.OwnerID().String()) - r.Response.Header.Set("X-Container-Id", obj.ContainerID().String()) + r.Response.Header.Set(hdrObjectID, obj.ID().String()) + r.Response.Header.Set(hdrOwnerID, obj.OwnerID().String()) + r.Response.Header.Set(hdrContainerID, obj.ContainerID().String()) if len(contentType) == 0 { if readDetector.err != nil { @@ -191,7 +191,7 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { } r.SetContentType(contentType) - r.Response.Header.Set("Content-Disposition", dis+"; filename="+path.Base(filename)) + r.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) } // systemBackwardTranslator is used to convert headers looking like '__NEOFS__ATTR_NAME' to 'Neofs-Attr-Name'. @@ -414,8 +414,8 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { return } - c.Response.Header.Set("Content-Type", "application/zip") - c.Response.Header.Set("Content-Disposition", "attachment; filename=\"archive.zip\"") + c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") + c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") c.Response.SetStatusCode(http.StatusOK) if err = d.streamFiles(c, containerID, ids); err != nil { diff --git a/downloader/head.go b/downloader/head.go index 8e113e6..6ec14e3 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -17,6 +17,12 @@ import ( const sizeToDetectType = 512 +const ( + hdrObjectID = "X-Object-Id" + hdrOwnerID = "X-Owner-Id" + hdrContainerID = "X-Container-Id" +) + func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { var start = time.Now() if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { @@ -33,7 +39,7 @@ func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { return } - r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) + r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10)) var contentType string for _, attr := range obj.Attributes() { key := attr.Key() @@ -52,14 +58,14 @@ func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { zap.Error(err)) continue } - r.Response.Header.Set("Last-Modified", time.Unix(value, 0).UTC().Format(http.TimeFormat)) + r.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat)) case object.AttributeContentType: contentType = val } } - r.Response.Header.Set("x-object-id", obj.ID().String()) - r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) - r.Response.Header.Set("x-container-id", obj.ContainerID().String()) + r.Response.Header.Set(hdrObjectID, obj.ID().String()) + r.Response.Header.Set(hdrOwnerID, obj.OwnerID().String()) + r.Response.Header.Set(hdrContainerID, obj.ContainerID().String()) if len(contentType) == 0 { objRange := object.NewRange() From 39c29c6d6a05f72a23c9730c3636b2076ea46a34 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 15 Feb 2022 12:13:43 +0300 Subject: [PATCH 235/548] [#125] Set object ids in a separate function Signed-off-by: Denis Kirillov --- downloader/download.go | 5 ++--- downloader/head.go | 11 ++++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 03b2729..4964eba 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -176,9 +176,8 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { contentType = val } } - r.Response.Header.Set(hdrObjectID, obj.ID().String()) - r.Response.Header.Set(hdrOwnerID, obj.OwnerID().String()) - r.Response.Header.Set(hdrContainerID, obj.ContainerID().String()) + + idsToResponse(&r.Response, obj) if len(contentType) == 0 { if readDetector.err != nil { diff --git a/downloader/head.go b/downloader/head.go index 6ec14e3..9093eb1 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -63,9 +63,8 @@ func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { contentType = val } } - r.Response.Header.Set(hdrObjectID, obj.ID().String()) - r.Response.Header.Set(hdrOwnerID, obj.OwnerID().String()) - r.Response.Header.Set(hdrContainerID, obj.ContainerID().String()) + + idsToResponse(&r.Response, obj) if len(contentType) == 0 { objRange := object.NewRange() @@ -86,6 +85,12 @@ func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { r.SetContentType(contentType) } +func idsToResponse(resp *fasthttp.Response, obj *object.Object) { + resp.Header.Set(hdrObjectID, obj.ID().String()) + resp.Header.Set(hdrOwnerID, obj.OwnerID().String()) + resp.Header.Set(hdrContainerID, obj.ContainerID().String()) +} + // HeadByAddress handles head requests using simple cid/oid format. func (d *Downloader) HeadByAddress(c *fasthttp.RequestCtx) { d.byAddress(c, request.headObject) From 2b7e4a36fbc40cb5f87b789b7895bca12e6c182c Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Tue, 8 Feb 2022 19:24:23 +0300 Subject: [PATCH 236/548] [#126] Upgrade NeoFS SDK Go library Core changes: - `object.ID` moved to new package `oid`; - `object.Address` moved to new package `address`; - `pool.Object` interface changes. Signed-off-by: Leonard Lyubich --- app.go | 2 - downloader/download.go | 262 +++++++++++++++++++++-------------------- downloader/head.go | 31 +++-- go.mod | 12 +- go.sum | 30 +++-- integration_test.go | 43 +++---- uploader/upload.go | 26 ++-- 7 files changed, 220 insertions(+), 186 deletions(-) diff --git a/app.go b/app.go index 267c92c..294c306 100644 --- a/app.go +++ b/app.go @@ -4,7 +4,6 @@ import ( "context" "crypto/ecdsa" "fmt" - "math" "strconv" "github.com/fasthttp/router" @@ -122,7 +121,6 @@ func newApp(ctx context.Context, opt ...Option) App { NodeConnectionTimeout: a.cfg.GetDuration(cfgConTimeout), NodeRequestTimeout: a.cfg.GetDuration(cfgReqTimeout), ClientRebalanceInterval: a.cfg.GetDuration(cfgRebalance), - SessionExpirationEpoch: math.MaxUint64, } a.pool, err = pb.Build(ctx, opts) if err != nil { diff --git a/downloader/download.go b/downloader/download.go index 4964eba..89ebb43 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -16,9 +16,10 @@ import ( "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" - "github.com/nspcc-dev/neofs-sdk-go/client" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/object/address" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -38,8 +39,6 @@ type ( log *zap.Logger } - objectIDs []*object.ID - errReader struct { data []byte err error @@ -116,39 +115,39 @@ func isValidValue(s string) bool { return true } -func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { +func (r request) receiveFile(clnt pool.Object, objectAddress *address.Address) { var ( err error dis = "inline" start = time.Now() filename string - obj *object.Object ) if err = tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) response.Error(r.RequestCtx, "could not fetch and store bearer token", fasthttp.StatusBadRequest) return } - readDetector := newDetector() - options := new(client.GetObjectParams). - WithAddress(objectAddress). - WithPayloadReaderHandler(func(reader io.Reader) { - readDetector.SetReader(reader) - readDetector.Detect() - }) - obj, err = clnt.GetObject(r.RequestCtx, options, bearerOpts(r.RequestCtx)) + rObj, err := clnt.GetObject(r.RequestCtx, *objectAddress, bearerOpts(r.RequestCtx)) if err != nil { r.handleNeoFSErr(err, start) return } + + // we can't close reader in this function, so how to do it? + if r.Request.URI().QueryArgs().GetBool("download") { dis = "attachment" } - r.Response.SetBodyStream(readDetector.MultiReader(), int(obj.PayloadSize())) - r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10)) + + readDetector := newDetector() + readDetector.SetReader(rObj.Payload) + readDetector.Detect() + + r.Response.SetBodyStream(readDetector.MultiReader(), int(rObj.Header.PayloadSize())) + r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(rObj.Header.PayloadSize(), 10)) var contentType string - for _, attr := range obj.Attributes() { + for _, attr := range rObj.Header.Attributes() { key := attr.Key() val := attr.Value() if !isValidToken(key) || !isValidValue(val) { @@ -177,7 +176,7 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { } } - idsToResponse(&r.Response, obj) + idsToResponse(&r.Response, &rObj.Header) if len(contentType) == 0 { if readDetector.err != nil { @@ -244,14 +243,6 @@ func (r *request) handleNeoFSErr(err error, start time.Time) { response.Error(r.RequestCtx, msg, code) } -func (o objectIDs) Slice() []string { - res := make([]string, 0, len(o)) - for _, oid := range o { - res = append(res, oid.String()) - } - return res -} - // Downloader is a download request handler. type Downloader struct { log *zap.Logger @@ -287,21 +278,21 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { // byAddress is wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, pool.Object, *object.Address)) { +func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, pool.Object, *address.Address)) { var ( - address = object.NewAddress() - cid, _ = c.UserValue("cid").(string) - oid, _ = c.UserValue("oid").(string) - val = strings.Join([]string{cid, oid}, "/") - log = d.log.With(zap.String("cid", cid), zap.String("oid", oid)) + addr = address.NewAddress() + idCnr, _ = c.UserValue("cid").(string) + idObj, _ = c.UserValue("oid").(string) + val = strings.Join([]string{idCnr, idObj}, "/") + log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj)) ) - if err := address.Parse(val); err != nil { + if err := addr.Parse(val); err != nil { log.Error("wrong object address", zap.Error(err)) response.Error(c, "wrong object address", fasthttp.StatusBadRequest) return } - f(*d.newRequest(c, log), d.pool, address) + f(*d.newRequest(c, log), d.pool, addr) } // DownloadByAttribute handles attribute-based download requests. @@ -310,7 +301,7 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { } // byAttribute is wrapper similar to byAddress. -func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Object, *object.Address)) { +func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Object, *address.Address)) { var ( httpStatus = fasthttp.StatusBadRequest scid, _ = c.UserValue("cid").(string) @@ -325,67 +316,47 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Ob return } - address, err := d.searchObject(c, log, containerID, key, val) + res, err := d.search(c, containerID, key, val, object.MatchStringEqual) if err != nil { - log.Error("couldn't search object", zap.Error(err)) - if errors.Is(err, errObjectNotFound) { - httpStatus = fasthttp.StatusNotFound - } - response.Error(c, "couldn't search object", httpStatus) + log.Error("could not search for objects", zap.Error(err)) + response.Error(c, "could not search for objects", fasthttp.StatusBadRequest) return } - f(*d.newRequest(c, log), d.pool, address) -} + defer res.Close() -func (d *Downloader) searchObject(c *fasthttp.RequestCtx, log *zap.Logger, cid *cid.ID, key, val string) (*object.Address, error) { - ids, err := d.searchByAttr(c, cid, key, val) - if err != nil { - return nil, err - } - if len(ids) > 1 { - log.Debug("found multiple objects", - zap.Strings("object_ids", objectIDs(ids).Slice()), - zap.Stringer("show_object_id", ids[0])) + buf := make([]oid.ID, 1) + + n, err := res.Read(buf) + if n == 0 { + if errors.Is(err, io.EOF) { + log.Error("object not found", zap.Error(err)) + response.Error(c, "object not found", fasthttp.StatusNotFound) + return + } + + log.Error("read object list failed", zap.Error(err)) + response.Error(c, "read object list failed", fasthttp.StatusBadRequest) + return } - return formAddress(cid, ids[0]), nil + var addrObj address.Address + addrObj.SetContainerID(containerID) + addrObj.SetObjectID(&buf[0]) + + f(*d.newRequest(c, log), d.pool, &addrObj) } -func formAddress(cid *cid.ID, oid *object.ID) *object.Address { - address := object.NewAddress() - address.SetContainerID(cid) - address.SetObjectID(oid) - return address -} +func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) (*pool.ResObjectSearch, error) { + filters := object.NewSearchFilters() + filters.AddRootFilter() + filters.AddFilter(key, val, op) -func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) ([]*object.ID, error) { - options := object.NewSearchFilters() - options.AddRootFilter() - options.AddFilter(key, val, op) - - sops := new(client.SearchObjectParams).WithContainerID(cid).WithSearchFilters(options) - ids, err := d.pool.SearchObject(c, sops) - if err != nil { - return nil, err - } - if len(ids) == 0 { - return nil, errObjectNotFound - } - return ids, nil -} - -func (d *Downloader) searchByPrefix(c *fasthttp.RequestCtx, cid *cid.ID, val string) ([]*object.ID, error) { - return d.search(c, cid, object.AttributeFileName, val, object.MatchCommonPrefix) -} - -func (d *Downloader) searchByAttr(c *fasthttp.RequestCtx, cid *cid.ID, key, val string) ([]*object.ID, error) { - return d.search(c, cid, key, val, object.MatchStringEqual) + return d.pool.SearchObjects(c, *cid, filters) } // DownloadZipped handles zip by prefix requests. func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { - status := fasthttp.StatusBadRequest scid, _ := c.UserValue("cid").(string) prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string)) log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) @@ -393,7 +364,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { containerID := cid.New() if err := containerID.Parse(scid); err != nil { log.Error("wrong container id", zap.Error(err)) - response.Error(c, "wrong container id", status) + response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } @@ -403,71 +374,110 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { return } - ids, err := d.searchByPrefix(c, containerID, prefix) + resSearch, err := d.search(c, containerID, object.AttributeFileName, prefix, object.MatchCommonPrefix) if err != nil { - log.Error("couldn't find objects", zap.Error(err)) - if errors.Is(err, errObjectNotFound) { - status = fasthttp.StatusNotFound - } - response.Error(c, "couldn't find objects", status) + log.Error("could not search for objects", zap.Error(err)) + response.Error(c, "could not search for objects", fasthttp.StatusBadRequest) return } + defer resSearch.Close() + c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") c.Response.SetStatusCode(http.StatusOK) - if err = d.streamFiles(c, containerID, ids); err != nil { - log.Error("couldn't stream files", zap.Error(err)) - response.Error(c, "couldn't stream", fasthttp.StatusInternalServerError) - return - } -} - -func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error { zipWriter := zip.NewWriter(c) compression := zip.Store if d.settings.ZipCompression { compression = zip.Deflate } - for _, id := range ids { - var r io.Reader - readerInitCtx, initReader := context.WithCancel(c) - options := new(client.GetObjectParams). - WithAddress(formAddress(cid, id)). - WithPayloadReaderHandler(func(reader io.Reader) { - r = reader - initReader() + var ( + addr address.Address + resGet *pool.ResGetObject + w io.Writer + bufZip []byte + ) + + addr.SetContainerID(containerID) + + optBearer := bearerOpts(c) + empty := true + n := 0 + buf := make([]oid.ID, 10) // configure? + +iterator: + for { + n, err = resSearch.Read(buf) + if err != nil { + if errors.Is(err, io.EOF) { + if empty { + log.Error("objects not found", zap.Error(err)) + response.Error(c, "objects not found", fasthttp.StatusNotFound) + return + } + + err = nil + + break + } + + log.Error("read object list failed", zap.Error(err)) + response.Error(c, "read object list failed", fasthttp.StatusBadRequest) // maybe best effort? + return + } + + if empty { + bufZip = make([]byte, 1024) // configure? + } + + empty = false + + for i := range buf[:n] { + addr.SetObjectID(&buf[i]) + + resGet, err = d.pool.GetObject(c, addr, optBearer) + if err != nil { + err = fmt.Errorf("get NeoFS object: %v", err) + break iterator + } + + w, err = zipWriter.CreateHeader(&zip.FileHeader{ + Name: getFilename(&resGet.Header), + Method: compression, + Modified: time.Now(), }) + if err != nil { + err = fmt.Errorf("zip create header: %v", err) + break iterator + } - obj, err := d.pool.GetObject(c, options, bearerOpts(c)) - if err != nil { - return err - } + _, err = io.CopyBuffer(w, resGet.Payload, bufZip) + if err != nil { + err = fmt.Errorf("copy object payload to zip file: %v", err) + break iterator + } - header := &zip.FileHeader{ - Name: getFilename(obj), - Method: compression, - Modified: time.Now(), - } - entryWriter, err := zipWriter.CreateHeader(header) - if err != nil { - return err - } + _ = resGet.Payload.Close() - <-readerInitCtx.Done() - _, err = io.Copy(entryWriter, r) - if err != nil { - return err - } - - if err = zipWriter.Flush(); err != nil { - return err + err = zipWriter.Flush() + if err != nil { + err = fmt.Errorf("flush zip writer: %v", err) + break iterator + } } } - return zipWriter.Close() + if err == nil { + err = zipWriter.Close() + } + + if err != nil { + log.Error("file streaming failure", zap.Error(err)) + response.Error(c, "file streaming failure", fasthttp.StatusInternalServerError) + return + } } func getFilename(obj *object.Object) string { diff --git a/downloader/head.go b/downloader/head.go index 9093eb1..7d9fdae 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -1,6 +1,7 @@ package downloader import ( + "io" "net/http" "strconv" "time" @@ -8,8 +9,8 @@ import ( "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" - "github.com/nspcc-dev/neofs-sdk-go/client" "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/object/address" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -23,7 +24,7 @@ const ( hdrContainerID = "X-Container-Id" ) -func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { +func (r request) headObject(clnt pool.Object, objectAddress *address.Address) { var start = time.Now() if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) @@ -31,9 +32,8 @@ func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { return } - options := new(client.ObjectHeaderParams).WithAddress(objectAddress) bearerOpt := bearerOpts(r.RequestCtx) - obj, err := clnt.GetObjectHeader(r.RequestCtx, options, bearerOpt) + obj, err := clnt.HeadObject(r.RequestCtx, *objectAddress, bearerOpt) if err != nil { r.handleNeoFSErr(err, start) return @@ -67,15 +67,22 @@ func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { idsToResponse(&r.Response, obj) if len(contentType) == 0 { - objRange := object.NewRange() - objRange.SetOffset(0) - if sizeToDetectType < obj.PayloadSize() { - objRange.SetLength(sizeToDetectType) - } else { - objRange.SetLength(obj.PayloadSize()) + sz := obj.PayloadSize() + if sz > sizeToDetectType { + sz = sizeToDetectType } - ops := new(client.RangeDataParams).WithAddress(objectAddress).WithRange(objRange) - data, err := clnt.ObjectPayloadRangeData(r.RequestCtx, ops, bearerOpt) + + res, err := clnt.ObjectRange(r.RequestCtx, *objectAddress, 0, sz, bearerOpt) + if err != nil { + r.handleNeoFSErr(err, start) + return + } + + defer res.Close() + + data := make([]byte, sz) // sync-pool it? + + _, err = io.ReadFull(res, data) if err != nil { r.handleNeoFSErr(err, start) return diff --git a/go.mod b/go.mod index 256f6b9..33310c6 100644 --- a/go.mod +++ b/go.mod @@ -7,22 +7,28 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/fasthttp/router v1.4.1 github.com/golang/mock v1.6.0 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/klauspost/compress v1.13.1 // indirect github.com/nspcc-dev/neo-go v0.98.0 - github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1 - github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477 + github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 github.com/prometheus/procfs v0.7.1 // indirect github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/testcontainers/testcontainers-go v0.12.0 github.com/valyala/fasthttp v1.28.0 go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.18.1 + golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf // indirect + golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect golang.org/x/tools v0.1.5 // indirect google.golang.org/grpc v1.41.0 + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 044e645..f90fd90 100644 --- a/go.sum +++ b/go.sum @@ -429,8 +429,9 @@ github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3i github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -455,8 +456,9 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -586,14 +588,16 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0 h1:yyW4sgY88/pLf0949qmgfkQXzRKC3CI/WyhqXNnwMd8= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= -github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1 h1:CGA56mhLLduWRuMHcWujP5Ek+gAnXHk0WuIWkG65G1s= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= +github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5 h1:y9tbmUYhcr052QXsa4/IfUKAi2cx3TGDsEZUAow3P/Y= +github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477 h1:JC+jt4ARpMV/L3OqPHBKxAmbMabU7RYl/L4KgBz3yPs= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02 h1:g9tIrZU45dVFUSiY7Bb8m43rV/CJiIoPgQrxnbtKfKE= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02/go.mod h1:NeDPJaKJ6yCOWXRmfc3aRrhBPEOeAPD7q/6bp1UQCbs= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= @@ -763,8 +767,9 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/testcontainers/testcontainers-go v0.12.0 h1:SK0NryGHIx7aifF6YqReORL18aGAA4bsDPtikDVCEyg= github.com/testcontainers/testcontainers-go v0.12.0/go.mod h1:SIndOQXZng0IW8iWU1Js0ynrfZ8xcxrTtDfF6rD2pxs= @@ -854,8 +859,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf h1:gdgmgieTI2lLaGI2N+xEiaCMUgo2XFmAS0rlF8HZoso= +golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -944,8 +950,9 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211108170745-6635138e15ea h1:FosBMXtOc8Tp9Hbo4ltl1WJSrTVewZU8MPnTPY2HdH8= golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1063,8 +1070,9 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3 h1:T6tyxxvHMj2L1R2kZg0uNMpS8ZhB9lRa9XRGTCSA65w= golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc= +golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= @@ -1082,8 +1090,9 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1272,8 +1281,9 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/integration_test.go b/integration_test.go index ffcd92f..4866fc7 100644 --- a/integration_test.go +++ b/integration_test.go @@ -16,10 +16,11 @@ import ( "time" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neofs-sdk-go/client" "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/object/address" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/policy" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/spf13/viper" @@ -108,21 +109,25 @@ func simplePut(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid err = CID.Parse(addr.CID) require.NoError(t, err) - oid := object.NewID() - err = oid.Parse(addr.OID) + id := oid.NewID() + err = id.Parse(addr.OID) require.NoError(t, err) - objectAddress := object.NewAddress() + objectAddress := address.NewAddress() objectAddress.SetContainerID(CID) - objectAddress.SetObjectID(oid) + objectAddress.SetObjectID(id) payload := bytes.NewBuffer(nil) - ops := new(client.GetObjectParams).WithAddress(objectAddress).WithPayloadWriter(payload) - obj, err := clientPool.GetObject(ctx, ops) + + res, err := clientPool.GetObject(ctx, *objectAddress) require.NoError(t, err) + + _, err = io.Copy(payload, res.Payload) + require.NoError(t, err) + require.Equal(t, content, payload.String()) - for _, attribute := range obj.Attributes() { + for _, attribute := range res.Header.Attributes() { require.Equal(t, attributes[attribute.Key()], attribute.Value()) } } @@ -133,9 +138,9 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid "some-attr": "some-get-value", } - oid := putObject(ctx, t, clientPool, CID, content, attributes) + id := putObject(ctx, t, clientPool, CID, content, attributes) - resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + oid.String()) + resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + id.String()) require.NoError(t, err) defer func() { err = resp.Body.Close() @@ -156,11 +161,11 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid content := "content of file" attributes := map[string]string{keyAttr: valAttr} - oid := putObject(ctx, t, clientPool, CID, content, attributes) + id := putObject(ctx, t, clientPool, CID, content, attributes) expectedAttr := map[string]string{ "X-Attribute-" + keyAttr: valAttr, - "x-object-id": oid.String(), + "x-object-id": id.String(), "x-container-id": CID.String(), } @@ -271,10 +276,9 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) pool.Pool pb.AddNode("localhost:8080", 1, 1) opts := &pool.BuilderOptions{ - Key: &key.PrivateKey, - NodeConnectionTimeout: 5 * time.Second, - NodeRequestTimeout: 5 * time.Second, - SessionExpirationEpoch: math.MaxUint64, + Key: &key.PrivateKey, + NodeConnectionTimeout: 5 * time.Second, + NodeRequestTimeout: 5 * time.Second, } clientPool, err := pb.Build(ctx, opts) require.NoError(t, err) @@ -305,7 +309,7 @@ func createContainer(ctx context.Context, t *testing.T, clientPool pool.Pool) *c return CID } -func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID, content string, attributes map[string]string) *object.ID { +func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID, content string, attributes map[string]string) *oid.ID { rawObject := object.NewRaw() rawObject.SetContainerID(CID) rawObject.SetOwnerID(clientPool.OwnerID()) @@ -319,9 +323,8 @@ func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid } rawObject.SetAttributes(attrs...) - ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(bytes.NewBufferString(content)) - oid, err := clientPool.PutObject(ctx, ops) + id, err := clientPool.PutObject(ctx, *rawObject.Object(), bytes.NewBufferString(content)) require.NoError(t, err) - return oid + return id } diff --git a/uploader/upload.go b/uploader/upload.go index 1861516..f6ed91e 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -17,6 +17,8 @@ import ( cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/object/address" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/owner" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/nspcc-dev/neofs-sdk-go/token" @@ -53,9 +55,9 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { var ( err error file MultipartFile - obj *object.ID - addr = object.NewAddress() - cid = cid.New() + obj *oid.ID + addr = address.NewAddress() + idCnr = cid.New() scid, _ = c.UserValue("cid").(string) log = u.log.With(zap.String("cid", scid)) bodyStream = c.RequestBodyStream() @@ -66,7 +68,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest) return } - if err = cid.Parse(scid); err != nil { + if err = idCnr.Parse(scid); err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return @@ -127,23 +129,21 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) attributes = append(attributes, timestamp) } - oid, bt := u.fetchOwnerAndBearerToken(c) + id, bt := u.fetchOwnerAndBearerToken(c) rawObject := object.NewRaw() - rawObject.SetContainerID(cid) - rawObject.SetOwnerID(oid) + rawObject.SetContainerID(idCnr) + rawObject.SetOwnerID(id) rawObject.SetAttributes(attributes...) - ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(file) - - if obj, err = u.pool.PutObject(c, ops, pool.WithBearer(bt)); err != nil { + if obj, err = u.pool.PutObject(c, *rawObject.Object(), file, pool.WithBearer(bt)); err != nil { log.Error("could not store file in neofs", zap.Error(err)) response.Error(c, "could not store file in neofs", fasthttp.StatusBadRequest) return } addr.SetObjectID(obj) - addr.SetContainerID(cid) + addr.SetContainerID(idCnr) // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { @@ -181,7 +181,7 @@ type putResponse struct { ContainerID string `json:"container_id"` } -func newPutResponse(addr *object.Address) *putResponse { +func newPutResponse(addr *address.Address) *putResponse { return &putResponse{ ObjectID: addr.ObjectID().String(), ContainerID: addr.ContainerID().String(), @@ -197,7 +197,7 @@ func (pr *putResponse) encode(w io.Writer) error { func getEpochDurations(ctx context.Context, p pool.Pool) (*epochDurations, error) { if conn, _, err := p.Connection(); err != nil { return nil, err - } else if networkInfoRes, err := conn.NetworkInfo(ctx); err != nil { + } else if networkInfoRes, err := conn.NetworkInfo(ctx, client.PrmNetworkInfo{}); err != nil { return nil, err } else if err = apistatus.ErrFromStatus(networkInfoRes.Status()); err != nil { return nil, err From c482bbd25a91684ac827b7da0b9991dbf0e09617 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Tue, 1 Mar 2022 23:38:03 +0300 Subject: [PATCH 237/548] [#126] downloader: Replace `Read` with `Iterate` on `ObjectListReader` Make `Downloader.DownloadZipped` to call `Iterate` method instead of `Read` one during processing the `ObjectListReader`. Signed-off-by: Leonard Lyubich --- downloader/download.go | 96 +++++++++++++++++++----------------------- go.mod | 4 +- go.sum | 8 ++-- integration_test.go | 13 +++--- 4 files changed, 56 insertions(+), 65 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 89ebb43..dd810f1 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -404,29 +404,10 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { optBearer := bearerOpts(c) empty := true - n := 0 - buf := make([]oid.ID, 10) // configure? + called := false -iterator: - for { - n, err = resSearch.Read(buf) - if err != nil { - if errors.Is(err, io.EOF) { - if empty { - log.Error("objects not found", zap.Error(err)) - response.Error(c, "objects not found", fasthttp.StatusNotFound) - return - } - - err = nil - - break - } - - log.Error("read object list failed", zap.Error(err)) - response.Error(c, "read object list failed", fasthttp.StatusBadRequest) // maybe best effort? - return - } + errIter := resSearch.Iterate(func(id oid.ID) bool { + called = true if empty { bufZip = make([]byte, 1024) // configure? @@ -434,39 +415,48 @@ iterator: empty = false - for i := range buf[:n] { - addr.SetObjectID(&buf[i]) + addr.SetObjectID(&id) - resGet, err = d.pool.GetObject(c, addr, optBearer) - if err != nil { - err = fmt.Errorf("get NeoFS object: %v", err) - break iterator - } - - w, err = zipWriter.CreateHeader(&zip.FileHeader{ - Name: getFilename(&resGet.Header), - Method: compression, - Modified: time.Now(), - }) - if err != nil { - err = fmt.Errorf("zip create header: %v", err) - break iterator - } - - _, err = io.CopyBuffer(w, resGet.Payload, bufZip) - if err != nil { - err = fmt.Errorf("copy object payload to zip file: %v", err) - break iterator - } - - _ = resGet.Payload.Close() - - err = zipWriter.Flush() - if err != nil { - err = fmt.Errorf("flush zip writer: %v", err) - break iterator - } + resGet, err = d.pool.GetObject(c, addr, optBearer) + if err != nil { + err = fmt.Errorf("get NeoFS object: %v", err) + return true } + + w, err = zipWriter.CreateHeader(&zip.FileHeader{ + Name: getFilename(&resGet.Header), + Method: compression, + Modified: time.Now(), + }) + if err != nil { + err = fmt.Errorf("zip create header: %v", err) + return true + } + + _, err = io.CopyBuffer(w, resGet.Payload, bufZip) + if err != nil { + err = fmt.Errorf("copy object payload to zip file: %v", err) + return true + } + + _ = resGet.Payload.Close() + + err = zipWriter.Flush() + if err != nil { + err = fmt.Errorf("flush zip writer: %v", err) + return true + } + + return false + }) + if errIter != nil { + log.Error("iterating over selected objects failed", zap.Error(errIter)) + response.Error(c, "iterating over selected objects", fasthttp.StatusBadRequest) + return + } else if !called { + log.Error("objects not found") + response.Error(c, "objects not found", fasthttp.StatusNotFound) + return } if err == nil { diff --git a/go.mod b/go.mod index 33310c6..f838189 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,8 @@ require ( github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/klauspost/compress v1.13.1 // indirect github.com/nspcc-dev/neo-go v0.98.0 - github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02 + github.com/nspcc-dev/neofs-api-go/v2 v2.12.0 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220228071935-07817fb4032d github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 github.com/prometheus/procfs v0.7.1 // indirect diff --git a/go.sum b/go.sum index f90fd90..051c241 100644 --- a/go.sum +++ b/go.sum @@ -589,15 +589,15 @@ github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1: github.com/nspcc-dev/neo-go v0.98.0 h1:yyW4sgY88/pLf0949qmgfkQXzRKC3CI/WyhqXNnwMd8= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5 h1:y9tbmUYhcr052QXsa4/IfUKAi2cx3TGDsEZUAow3P/Y= -github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= +github.com/nspcc-dev/neofs-api-go/v2 v2.12.0 h1:xWqXzorDk9WFMTtWP7cwwlyJDL1X6Z4HT1e5zqkq7xY= +github.com/nspcc-dev/neofs-api-go/v2 v2.12.0/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02 h1:g9tIrZU45dVFUSiY7Bb8m43rV/CJiIoPgQrxnbtKfKE= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02/go.mod h1:NeDPJaKJ6yCOWXRmfc3aRrhBPEOeAPD7q/6bp1UQCbs= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220228071935-07817fb4032d h1:ku9s0XJ2LoWbB6nUjkyP7M8ki2nLOlMIvi4fAocf+iY= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220228071935-07817fb4032d/go.mod h1:/WV31AQHs6YLTjMgMjMZw8Z3/Q7b6kMjNgJVsRab5AU= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/integration_test.go b/integration_test.go index 4866fc7..bac23bf 100644 --- a/integration_test.go +++ b/integration_test.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "io" - "math" "mime/multipart" "net/http" "sort" @@ -47,7 +46,8 @@ func TestIntegration(t *testing.T) { aioContainer := createDockerContainer(ctx, t, aioImage+version) cancel := runServer() clientPool := getPool(ctx, t, key) - CID := createContainer(ctx, t, clientPool) + CID, err := createContainer(ctx, t, clientPool) + require.NoError(t, err, version) t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) }) t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID) }) @@ -285,7 +285,7 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) pool.Pool return clientPool } -func createContainer(ctx context.Context, t *testing.T, clientPool pool.Pool) *cid.ID { +func createContainer(ctx context.Context, t *testing.T, clientPool pool.Pool) (*cid.ID, error) { pp, err := policy.Parse("REP 1") require.NoError(t, err) @@ -297,16 +297,17 @@ func createContainer(ctx context.Context, t *testing.T, clientPool pool.Pool) *c cnr.SetOwnerID(clientPool.OwnerID()) CID, err := clientPool.PutContainer(ctx, cnr) - require.NoError(t, err) + if err != nil { + return nil, err + } fmt.Println(CID.String()) err = clientPool.WaitForContainerPresence(ctx, CID, &pool.ContainerPollingParams{ CreationTimeout: 15 * time.Second, PollInterval: 3 * time.Second, }) - require.NoError(t, err) - return CID + return CID, err } func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID, content string, attributes map[string]string) *oid.ID { From 03985f2eda5874b4caaf1e4e061c99155162718a Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Wed, 2 Mar 2022 13:46:11 +0300 Subject: [PATCH 238/548] [#126] Upgrade NeoFS SDK Go to 2nd release candidate v1.0.0 Avoid using the deprecated elements. Signed-off-by: Leonard Lyubich --- go.mod | 4 ++-- go.sum | 8 ++++---- integration_test.go | 10 +++++----- uploader/upload.go | 14 +++++++------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index f838189..7de7306 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,8 @@ require ( github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/klauspost/compress v1.13.1 // indirect github.com/nspcc-dev/neo-go v0.98.0 - github.com/nspcc-dev/neofs-api-go/v2 v2.12.0 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220228071935-07817fb4032d + github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220302134950-d065453bd0a7 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.2 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 github.com/prometheus/procfs v0.7.1 // indirect diff --git a/go.sum b/go.sum index 051c241..dd4124f 100644 --- a/go.sum +++ b/go.sum @@ -589,15 +589,15 @@ github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1: github.com/nspcc-dev/neo-go v0.98.0 h1:yyW4sgY88/pLf0949qmgfkQXzRKC3CI/WyhqXNnwMd8= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.12.0 h1:xWqXzorDk9WFMTtWP7cwwlyJDL1X6Z4HT1e5zqkq7xY= -github.com/nspcc-dev/neofs-api-go/v2 v2.12.0/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= +github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220302134950-d065453bd0a7 h1:hLMvj4K9djzBg+TaeDGQWGuohzXvcThi0r0LSLhhi3M= +github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220302134950-d065453bd0a7/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220228071935-07817fb4032d h1:ku9s0XJ2LoWbB6nUjkyP7M8ki2nLOlMIvi4fAocf+iY= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220228071935-07817fb4032d/go.mod h1:/WV31AQHs6YLTjMgMjMZw8Z3/Q7b6kMjNgJVsRab5AU= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.2 h1:m+em1eyrYFIGUdzs2asDCJH0GVWH+9rYdjLTO42mxSY= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.2/go.mod h1:bkVH6yZXH5/F2Sut20SDOlQbveBlaVJ0vOX20tLGnZw= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/integration_test.go b/integration_test.go index bac23bf..8c124e4 100644 --- a/integration_test.go +++ b/integration_test.go @@ -311,9 +311,9 @@ func createContainer(ctx context.Context, t *testing.T, clientPool pool.Pool) (* } func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID, content string, attributes map[string]string) *oid.ID { - rawObject := object.NewRaw() - rawObject.SetContainerID(CID) - rawObject.SetOwnerID(clientPool.OwnerID()) + obj := object.New() + obj.SetContainerID(CID) + obj.SetOwnerID(clientPool.OwnerID()) var attrs []*object.Attribute for key, val := range attributes { @@ -322,9 +322,9 @@ func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid attr.SetValue(val) attrs = append(attrs, attr) } - rawObject.SetAttributes(attrs...) + obj.SetAttributes(attrs...) - id, err := clientPool.PutObject(ctx, *rawObject.Object(), bytes.NewBufferString(content)) + id, err := clientPool.PutObject(ctx, *obj, bytes.NewBufferString(content)) require.NoError(t, err) return id diff --git a/uploader/upload.go b/uploader/upload.go index f6ed91e..c3ca85b 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -55,7 +55,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { var ( err error file MultipartFile - obj *oid.ID + idObj *oid.ID addr = address.NewAddress() idCnr = cid.New() scid, _ = c.UserValue("cid").(string) @@ -131,18 +131,18 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { } id, bt := u.fetchOwnerAndBearerToken(c) - rawObject := object.NewRaw() - rawObject.SetContainerID(idCnr) - rawObject.SetOwnerID(id) - rawObject.SetAttributes(attributes...) + obj := object.New() + obj.SetContainerID(idCnr) + obj.SetOwnerID(id) + obj.SetAttributes(attributes...) - if obj, err = u.pool.PutObject(c, *rawObject.Object(), file, pool.WithBearer(bt)); err != nil { + if idObj, err = u.pool.PutObject(c, *obj, file, pool.WithBearer(bt)); err != nil { log.Error("could not store file in neofs", zap.Error(err)) response.Error(c, "could not store file in neofs", fasthttp.StatusBadRequest) return } - addr.SetObjectID(obj) + addr.SetObjectID(idObj) addr.SetContainerID(idCnr) // Try to return the response, otherwise, if something went wrong, throw an error. From f55edbb613414b6da6d9daaf01e7c0604904c5e6 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 4 Mar 2022 11:03:40 +0300 Subject: [PATCH 239/548] [#130] Remove race condition check in integration test Race condition checks are suitable for unit tests but mostly useless in case of full-size integration test with neofs-all-in-one test container. Signed-off-by: Alex Vanin --- integration_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration_test.go b/integration_test.go index 8c124e4..3442116 100644 --- a/integration_test.go +++ b/integration_test.go @@ -1,3 +1,6 @@ +//go:build !race +// +build !race + package main import ( From 271451dc3211a6d8750f5e5d5e896945e04dd9b9 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Fri, 4 Mar 2022 09:53:49 +0300 Subject: [PATCH 240/548] [#128] downloader: Simplify detecting the Content-Type from payload Signed-off-by: Leonard Lyubich --- downloader/download.go | 133 +++++++++++++++++--------------------- downloader/head.go | 23 ++----- downloader/reader_test.go | 54 ++-------------- 3 files changed, 70 insertions(+), 140 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index dd810f1..1718db8 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -2,6 +2,7 @@ package downloader import ( "archive/zip" + "bytes" "context" "errors" "fmt" @@ -25,74 +26,13 @@ import ( "go.uber.org/zap" ) -type ( - detector struct { - io.Reader - err error - contentType string - done chan struct{} - data []byte - } - - request struct { - *fasthttp.RequestCtx - log *zap.Logger - } - - errReader struct { - data []byte - err error - offset int - } -) +type request struct { + *fasthttp.RequestCtx + log *zap.Logger +} var errObjectNotFound = errors.New("object not found") -func newReader(data []byte, err error) *errReader { - return &errReader{data: data, err: err} -} - -func (r *errReader) Read(b []byte) (int, error) { - if r.offset >= len(r.data) { - return 0, io.EOF - } - n := copy(b, r.data[r.offset:]) - r.offset += n - if r.offset >= len(r.data) { - return n, r.err - } - return n, nil -} - -const contentTypeDetectSize = 512 - -func newDetector() *detector { - return &detector{done: make(chan struct{}), data: make([]byte, contentTypeDetectSize)} -} - -func (d *detector) Wait() { - <-d.done -} - -func (d *detector) SetReader(reader io.Reader) { - d.Reader = reader -} - -func (d *detector) Detect() { - n, err := d.Reader.Read(d.data) - if err != nil && err != io.EOF { - d.err = err - return - } - d.data = d.data[:n] - d.contentType = http.DetectContentType(d.data) - close(d.done) -} - -func (d *detector) MultiReader() io.Reader { - return io.MultiReader(newReader(d.data, d.err), d.Reader) -} - func isValidToken(s string) bool { for _, c := range s { if c <= ' ' || c > 127 { @@ -115,6 +55,35 @@ func isValidValue(s string) bool { return true } +type readCloser struct { + io.Reader + io.Closer +} + +// initializes io.Reader with limited size and detects Content-Type from it. +// Returns r's error directly. Also returns processed data. +func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (string, []byte, error) { + if maxSize > sizeToDetectType { + maxSize = sizeToDetectType + } + + buf := make([]byte, maxSize) // maybe sync-pool the slice? + + r, err := rInit(maxSize) + if err != nil { + return "", nil, err + } + + n, err := r.Read(buf) + if err != nil && err != io.EOF { + return "", nil, err + } + + buf = buf[:n] + + return http.DetectContentType(buf), buf, err // to not lose io.EOF +} + func (r request) receiveFile(clnt pool.Object, objectAddress *address.Address) { var ( err error @@ -140,12 +109,9 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *address.Address) { dis = "attachment" } - readDetector := newDetector() - readDetector.SetReader(rObj.Payload) - readDetector.Detect() + payloadSize := rObj.Header.PayloadSize() - r.Response.SetBodyStream(readDetector.MultiReader(), int(rObj.Header.PayloadSize())) - r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(rObj.Header.PayloadSize(), 10)) + r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) var contentType string for _, attr := range rObj.Header.Attributes() { key := attr.Key() @@ -179,17 +145,34 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *address.Address) { idsToResponse(&r.Response, &rObj.Header) if len(contentType) == 0 { - if readDetector.err != nil { - r.log.Error("could not read object", zap.Error(err)) - response.Error(r.RequestCtx, "could not read object", fasthttp.StatusBadRequest) + // determine the Content-Type from the payload head + var payloadHead []byte + + contentType, payloadHead, err = readContentType(payloadSize, func(uint64) (io.Reader, error) { + return rObj.Payload, nil + }) + if err != nil && err != io.EOF { + r.log.Error("could not detect Content-Type from payload", zap.Error(err)) + response.Error(r.RequestCtx, "could not detect Content-Type from payload", fasthttp.StatusBadRequest) return } - readDetector.Wait() - contentType = readDetector.contentType + + // reset payload reader since part of the data has been read + var r io.Reader = bytes.NewReader(payloadHead) + + if err != io.EOF { // otherwise, we've already read full payload + r = io.MultiReader(r, rObj.Payload) + } + + // note: we could do with io.Reader, but SetBodyStream below closes body stream + // if it implements io.Closer and that's useful for us. + rObj.Payload = readCloser{r, rObj.Payload} } r.SetContentType(contentType) r.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) + + r.Response.SetBodyStream(rObj.Payload, int(payloadSize)) } // systemBackwardTranslator is used to convert headers looking like '__NEOFS__ATTR_NAME' to 'Neofs-Attr-Name'. diff --git a/downloader/head.go b/downloader/head.go index 7d9fdae..620d15d 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -16,6 +16,7 @@ import ( "go.uber.org/zap" ) +// max bytes needed to detect content type according to http.DetectContentType docs. const sizeToDetectType = 512 const ( @@ -67,27 +68,13 @@ func (r request) headObject(clnt pool.Object, objectAddress *address.Address) { idsToResponse(&r.Response, obj) if len(contentType) == 0 { - sz := obj.PayloadSize() - if sz > sizeToDetectType { - sz = sizeToDetectType - } - - res, err := clnt.ObjectRange(r.RequestCtx, *objectAddress, 0, sz, bearerOpt) - if err != nil { + contentType, _, err = readContentType(obj.PayloadSize(), func(sz uint64) (io.Reader, error) { + return clnt.ObjectRange(r.RequestCtx, *objectAddress, 0, sz, bearerOpt) + }) + if err != nil && err != io.EOF { r.handleNeoFSErr(err, start) return } - - defer res.Close() - - data := make([]byte, sz) // sync-pool it? - - _, err = io.ReadFull(res, data) - if err != nil { - r.handleNeoFSErr(err, start) - return - } - contentType = http.DetectContentType(data) } r.SetContentType(contentType) } diff --git a/downloader/reader_test.go b/downloader/reader_test.go index 96a42e1..8d58185 100644 --- a/downloader/reader_test.go +++ b/downloader/reader_test.go @@ -1,8 +1,6 @@ package downloader import ( - "bytes" - "fmt" "io" "strings" "testing" @@ -10,40 +8,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestReader(t *testing.T) { - data := []byte("test string") - err := fmt.Errorf("something wrong") - - for _, tc := range []struct { - err error - buff []byte - }{ - {err: nil, buff: make([]byte, len(data)+1)}, - {err: nil, buff: make([]byte, len(data))}, - {err: nil, buff: make([]byte, len(data)-1)}, - {err: err, buff: make([]byte, len(data)+1)}, - {err: err, buff: make([]byte, len(data))}, - {err: err, buff: make([]byte, len(data)-1)}, - } { - var res []byte - var err error - var n int - - r := newReader(data, tc.err) - for err == nil { - n, err = r.Read(tc.buff) - res = append(res, tc.buff[:n]...) - } - - if tc.err == nil { - require.Equal(t, io.EOF, err) - } else { - require.Equal(t, tc.err, err) - } - require.Equal(t, data, res) - } -} - func TestDetector(t *testing.T) { txtContentType := "text/plain; charset=utf-8" sb := strings.Builder{} @@ -68,19 +32,15 @@ func TestDetector(t *testing.T) { }, } { t.Run(tc.Name, func(t *testing.T) { - detector := newDetector() + contentType, data, err := readContentType(uint64(len(tc.Expected)), + func(sz uint64) (io.Reader, error) { + return strings.NewReader(tc.Expected), nil + }, + ) - go func() { - detector.SetReader(bytes.NewBufferString(tc.Expected)) - detector.Detect() - }() - - detector.Wait() - require.Equal(t, tc.ContentType, detector.contentType) - - data, err := io.ReadAll(detector.MultiReader()) require.NoError(t, err) - require.Equal(t, tc.Expected, string(data)) + require.Equal(t, tc.ContentType, contentType) + require.True(t, strings.HasPrefix(tc.Expected, string(data))) }) } } From f6ab94027c49ce3c7250d2f376017d489b1d9e95 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Fri, 4 Mar 2022 11:36:23 +0300 Subject: [PATCH 241/548] [#128] downloader: Avoid var naming collision Signed-off-by: Leonard Lyubich --- downloader/download.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 1718db8..da09223 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -158,15 +158,15 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *address.Address) { } // reset payload reader since part of the data has been read - var r io.Reader = bytes.NewReader(payloadHead) + var headReader io.Reader = bytes.NewReader(payloadHead) if err != io.EOF { // otherwise, we've already read full payload - r = io.MultiReader(r, rObj.Payload) + headReader = io.MultiReader(headReader, rObj.Payload) } // note: we could do with io.Reader, but SetBodyStream below closes body stream // if it implements io.Closer and that's useful for us. - rObj.Payload = readCloser{r, rObj.Payload} + rObj.Payload = readCloser{headReader, rObj.Payload} } r.SetContentType(contentType) From c4521300ac90cffc05fb8d3849622b82d078d071 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 10 Mar 2022 16:33:51 +0300 Subject: [PATCH 242/548] [#129] Update fasthttp to v1.34.0 Signed-off-by: Denis Kirillov --- go.mod | 7 +------ go.sum | 30 +++++++++++++++++------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 7de7306..dcb8245 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,11 @@ module github.com/nspcc-dev/neofs-http-gw go 1.16 require ( - github.com/andybalholm/brotli v1.0.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/fasthttp/router v1.4.1 github.com/golang/mock v1.6.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect - github.com/klauspost/compress v1.13.1 // indirect github.com/nspcc-dev/neo-go v0.98.0 github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220302134950-d065453bd0a7 github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.2 @@ -21,12 +19,9 @@ require ( github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/testcontainers/testcontainers-go v0.12.0 - github.com/valyala/fasthttp v1.28.0 + github.com/valyala/fasthttp v1.34.0 go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.18.1 - golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf // indirect - golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect - golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect golang.org/x/tools v0.1.5 // indirect google.golang.org/grpc v1.41.0 diff --git a/go.sum b/go.sum index dd4124f..d4f79dd 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:C github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= -github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1 h1:zFRi26YWd7NIorBXe8UkevRl0dIvk/AnXHWaAiZG+Yk= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= @@ -499,8 +499,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= -github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -784,8 +784,9 @@ github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.28.0 h1:ruVmTmZaBR5i67NqnjvvH5gEv0zwHfWtbjoyW98iho4= github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA= +github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= +github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= @@ -860,8 +861,8 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf h1:gdgmgieTI2lLaGI2N+xEiaCMUgo2XFmAS0rlF8HZoso= -golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -951,8 +952,9 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1071,12 +1073,13 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc= -golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1084,8 +1087,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From acd8bbb34b57a3519fdfe880725268889ff34b1b Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 11 Mar 2022 12:08:16 +0300 Subject: [PATCH 243/548] [#129] Fix race condition tests Signed-off-by: Denis Kirillov --- integration_test.go | 3 --- uploader/upload.go | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_test.go b/integration_test.go index 3442116..8c124e4 100644 --- a/integration_test.go +++ b/integration_test.go @@ -1,6 +1,3 @@ -//go:build !race -// +build !race - package main import ( diff --git a/uploader/upload.go b/uploader/upload.go index c3ca85b..5fffa00 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -136,7 +136,10 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { obj.SetOwnerID(id) obj.SetAttributes(attributes...) - if idObj, err = u.pool.PutObject(c, *obj, file, pool.WithBearer(bt)); err != nil { + ctx, cancel := context.WithCancel(c) + defer cancel() + + if idObj, err = u.pool.PutObject(ctx, *obj, file, pool.WithBearer(bt)); err != nil { log.Error("could not store file in neofs", zap.Error(err)) response.Error(c, "could not store file in neofs", fasthttp.StatusBadRequest) return From 6d362894ad2fa447afa0c666dc9232773e469336 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 16 Mar 2022 11:10:56 +0300 Subject: [PATCH 244/548] [#132] Update NeoFS SDK to v1.0.0-rc.3 Signed-off-by: Denis Kirillov --- app.go | 2 +- downloader/download.go | 10 +++++----- downloader/head.go | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- integration_test.go | 18 +++++++++--------- uploader/upload.go | 14 +++++++------- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/app.go b/app.go index 294c306..2e1d1a7 100644 --- a/app.go +++ b/app.go @@ -25,7 +25,7 @@ import ( type ( app struct { log *zap.Logger - pool pool.Pool + pool *pool.Pool cfg *viper.Viper auxiliaryLog logger.Logger webServer *fasthttp.Server diff --git a/downloader/download.go b/downloader/download.go index da09223..3599119 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -84,7 +84,7 @@ func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (str return http.DetectContentType(buf), buf, err // to not lose io.EOF } -func (r request) receiveFile(clnt pool.Object, objectAddress *address.Address) { +func (r request) receiveFile(clnt *pool.Pool, objectAddress *address.Address) { var ( err error dis = "inline" @@ -229,7 +229,7 @@ func (r *request) handleNeoFSErr(err error, start time.Time) { // Downloader is a download request handler. type Downloader struct { log *zap.Logger - pool pool.Pool + pool *pool.Pool settings Settings } @@ -238,7 +238,7 @@ type Settings struct { } // New creates an instance of Downloader using specified options. -func New(log *zap.Logger, settings Settings, conns pool.Pool) (*Downloader, error) { +func New(log *zap.Logger, settings Settings, conns *pool.Pool) (*Downloader, error) { var err error d := &Downloader{log: log, pool: conns, settings: settings} if err != nil { @@ -261,7 +261,7 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { // byAddress is wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, pool.Object, *address.Address)) { +func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) { var ( addr = address.NewAddress() idCnr, _ = c.UserValue("cid").(string) @@ -284,7 +284,7 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { } // byAttribute is wrapper similar to byAddress. -func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Object, *address.Address)) { +func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) { var ( httpStatus = fasthttp.StatusBadRequest scid, _ = c.UserValue("cid").(string) diff --git a/downloader/head.go b/downloader/head.go index 620d15d..ddcc770 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -25,7 +25,7 @@ const ( hdrContainerID = "X-Container-Id" ) -func (r request) headObject(clnt pool.Object, objectAddress *address.Address) { +func (r request) headObject(clnt *pool.Pool, objectAddress *address.Address) { var start = time.Now() if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) diff --git a/go.mod b/go.mod index dcb8245..65c4c71 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/gorilla/mux v1.8.0 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/nspcc-dev/neo-go v0.98.0 - github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220302134950-d065453bd0a7 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.2 + github.com/nspcc-dev/neofs-api-go/v2 v2.12.1 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 github.com/prometheus/procfs v0.7.1 // indirect diff --git a/go.sum b/go.sum index d4f79dd..ab2b770 100644 --- a/go.sum +++ b/go.sum @@ -589,15 +589,15 @@ github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1: github.com/nspcc-dev/neo-go v0.98.0 h1:yyW4sgY88/pLf0949qmgfkQXzRKC3CI/WyhqXNnwMd8= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220302134950-d065453bd0a7 h1:hLMvj4K9djzBg+TaeDGQWGuohzXvcThi0r0LSLhhi3M= -github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220302134950-d065453bd0a7/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= +github.com/nspcc-dev/neofs-api-go/v2 v2.12.1 h1:PVU2rLlG9S0jDe5eKyaUs4nKo/la+mN5pvz32Gib3qM= +github.com/nspcc-dev/neofs-api-go/v2 v2.12.1/go.mod h1:73j09Xa7I2zQbM3HCvAHnDHPYiiWnEHa1d6Z6RDMBLU= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.2 h1:m+em1eyrYFIGUdzs2asDCJH0GVWH+9rYdjLTO42mxSY= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.2/go.mod h1:bkVH6yZXH5/F2Sut20SDOlQbveBlaVJ0vOX20tLGnZw= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3 h1:ofaiKPYY67a0cQMF+YSChDO48SBQtWlpZnK++cAeqQM= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3/go.mod h1:0hTXmyJnbw8j4BR1oltN7mFIIrVp1IFLdh8qBzAR464= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/integration_test.go b/integration_test.go index 8c124e4..882c138 100644 --- a/integration_test.go +++ b/integration_test.go @@ -72,7 +72,7 @@ func runServer() context.CancelFunc { return cancel } -func simplePut(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID) { +func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { content := "content of file" keyAttr, valAttr := "User-Attribute", "user value" @@ -132,7 +132,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid } } -func simpleGet(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID) { +func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { content := "content of file" attributes := map[string]string{ "some-attr": "some-get-value", @@ -156,7 +156,7 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid } } -func getByAttr(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID) { +func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { keyAttr, valAttr := "some-attr", "some-get-by-attr-value" content := "content of file" attributes := map[string]string{keyAttr: valAttr} @@ -185,7 +185,7 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid } } -func getZip(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID) { +func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"} contents := []string{"content of file1", "content of file2"} attributes1 := map[string]string{object.AttributeFileName: names[0]} @@ -271,7 +271,7 @@ func getDefaultConfig() *viper.Viper { return v } -func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) pool.Pool { +func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool { pb := new(pool.Builder) pb.AddNode("localhost:8080", 1, 1) @@ -285,7 +285,7 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) pool.Pool return clientPool } -func createContainer(ctx context.Context, t *testing.T, clientPool pool.Pool) (*cid.ID, error) { +func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool) (*cid.ID, error) { pp, err := policy.Parse("REP 1") require.NoError(t, err) @@ -310,17 +310,17 @@ func createContainer(ctx context.Context, t *testing.T, clientPool pool.Pool) (* return CID, err } -func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID, content string, attributes map[string]string) *oid.ID { +func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, content string, attributes map[string]string) *oid.ID { obj := object.New() obj.SetContainerID(CID) obj.SetOwnerID(clientPool.OwnerID()) - var attrs []*object.Attribute + var attrs []object.Attribute for key, val := range attributes { attr := object.NewAttribute() attr.SetKey(key) attr.SetValue(val) - attrs = append(attrs, attr) + attrs = append(attrs, *attr) } obj.SetAttributes(attrs...) diff --git a/uploader/upload.go b/uploader/upload.go index 5fffa00..4b094a5 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -34,7 +34,7 @@ const ( // Uploader is an upload request handler. type Uploader struct { log *zap.Logger - pool pool.Pool + pool *pool.Pool enableDefaultTimestamp bool } @@ -46,7 +46,7 @@ type epochDurations struct { // New creates a new Uploader using specified logger, connection pool and // other options. -func New(log *zap.Logger, conns pool.Pool, enableDefaultTimestamp bool) *Uploader { +func New(log *zap.Logger, conns *pool.Pool, enableDefaultTimestamp bool) *Uploader { return &Uploader{log, conns, enableDefaultTimestamp} } @@ -107,27 +107,27 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { } } - attributes := make([]*object.Attribute, 0, len(filtered)) + attributes := make([]object.Attribute, 0, len(filtered)) // prepares attributes from filtered headers for key, val := range filtered { attribute := object.NewAttribute() attribute.SetKey(key) attribute.SetValue(val) - attributes = append(attributes, attribute) + attributes = append(attributes, *attribute) } // sets FileName attribute if it wasn't set from header if _, ok := filtered[object.AttributeFileName]; !ok { filename := object.NewAttribute() filename.SetKey(object.AttributeFileName) filename.SetValue(file.FileName()) - attributes = append(attributes, filename) + attributes = append(attributes, *filename) } // sets Timestamp attribute if it wasn't set from header and enabled by settings if _, ok := filtered[object.AttributeTimestamp]; !ok && u.enableDefaultTimestamp { timestamp := object.NewAttribute() timestamp.SetKey(object.AttributeTimestamp) timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) - attributes = append(attributes, timestamp) + attributes = append(attributes, *timestamp) } id, bt := u.fetchOwnerAndBearerToken(c) @@ -197,7 +197,7 @@ func (pr *putResponse) encode(w io.Writer) error { return enc.Encode(pr) } -func getEpochDurations(ctx context.Context, p pool.Pool) (*epochDurations, error) { +func getEpochDurations(ctx context.Context, p *pool.Pool) (*epochDurations, error) { if conn, _, err := p.Connection(); err != nil { return nil, err } else if networkInfoRes, err := conn.NetworkInfo(ctx, client.PrmNetworkInfo{}); err != nil { From f572aeb6daf3f00f4fd6b79c9c67a240bb5f7fa7 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 16 Mar 2022 11:40:33 +0300 Subject: [PATCH 245/548] Release v0.19.0 Signed-off-by: Denis Kirillov --- CHANGELOG.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb55c5e..2e3b1b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ This document outlines major changes between releases. +## [0.19.0] - 2022-03-16 + +### Fixed +- Uploading object with zero payload (#122) +- Different headers format in GET and HEAD (#125) +- Fixed project name in docs (#120) + +### Added +- Support object attributes with spaces (#123) + +### Changed +- Updated fasthttp to v1.34.0 (#129) +- Updated NeoFS SDK to v1.0.0-rc.3 (#126, #132) +- Refactored content type detecting (#128) + + ## [0.18.0] - 2021-12-10 ### Fixed @@ -128,4 +144,5 @@ releases](https://github.com/nspcc-dev/neofs-http-gw/releases/) for older releases. [0.17.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.16.1...v0.17.0 -[0.18.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.17.0...v0.18.0 \ No newline at end of file +[0.18.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.17.0...v0.18.0 +[0.19.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.18.0...v0.19.0 From d1bdef6f83814134048bb41c3aa2044fcac63696 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 21 Mar 2022 12:10:02 +0300 Subject: [PATCH 246/548] [#133] Replace `go get` with `go install` Since Go 1.16, go install can install a command at a version specified on the command line while. Starting in Go 1.17, installing executables with go get is deprecated. Signed-off-by: Alex Vanin --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fd87190..866e715 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ NeoFS HTTP Gateway bridges NeoFS internal protocol and HTTP standard. ## Installation -```go get -u github.com/nspcc-dev/neofs-http-gw``` +```go install github.com/nspcc-dev/neofs-http-gw``` Or you can call `make` to build it from the cloned repository (the binary will end up in `bin/neofs-http-gw`). From d891c13cb3960cfee3be12d2aaf60cab98c2bc49 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 25 Mar 2022 16:06:33 +0300 Subject: [PATCH 247/548] [#137] Drop sdk logger Signed-off-by: Denis Kirillov --- app.go | 22 ++++++---------- go.mod | 1 - main.go | 57 ++++++++++++++++++++++++++++------------- metrics.go | 21 +++++---------- settings.go | 30 +++------------------- uploader/filter_test.go | 5 ++-- 6 files changed, 60 insertions(+), 76 deletions(-) diff --git a/app.go b/app.go index 2e1d1a7..63334cd 100644 --- a/app.go +++ b/app.go @@ -14,22 +14,19 @@ import ( "github.com/nspcc-dev/neofs-http-gw/downloader" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/uploader" - "github.com/nspcc-dev/neofs-sdk-go/logger" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" - "google.golang.org/grpc/grpclog" ) type ( app struct { - log *zap.Logger - pool *pool.Pool - cfg *viper.Viper - auxiliaryLog logger.Logger - webServer *fasthttp.Server - webDone chan struct{} + log *zap.Logger + pool *pool.Pool + cfg *viper.Viper + webServer *fasthttp.Server + webDone chan struct{} } // App is an interface for the main gateway function. @@ -77,10 +74,7 @@ func newApp(ctx context.Context, opt ...Option) App { for i := range opt { opt[i](a) } - a.auxiliaryLog = logger.GRPC(a.log) - if a.cfg.GetBool(cmdVerbose) { - grpclog.SetLoggerV2(a.auxiliaryLog) - } + // -- setup FastHTTP server -- a.webServer.Name = "neofs-http-gw" a.webServer.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) @@ -182,7 +176,7 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*ecds } func (a *app) Wait() { - a.log.Info("starting application", zap.String("version", a.cfg.GetString(cfgApplicationVersion))) + a.log.Info("starting application", zap.String("app_name", "neofs-http-gw"), zap.String("version", Version)) <-a.webDone // wait for web-server to be stopped } @@ -220,7 +214,7 @@ func (a *app) Serve(ctx context.Context) { // enable metrics if a.cfg.GetBool(cmdMetrics) { a.log.Info("added path /metrics/") - attachMetrics(r, a.auxiliaryLog) + attachMetrics(r, a.log) } // enable pprof if a.cfg.GetBool(cmdPprof) { diff --git a/go.mod b/go.mod index 65c4c71..046ec64 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,5 @@ require ( go.uber.org/zap v1.18.1 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect golang.org/x/tools v0.1.5 // indirect - google.golang.org/grpc v1.41.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/main.go b/main.go index 966904c..b54fd67 100644 --- a/main.go +++ b/main.go @@ -2,12 +2,13 @@ package main import ( "context" + "fmt" "os/signal" "syscall" - "github.com/nspcc-dev/neofs-sdk-go/logger" "github.com/spf13/viper" "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) func main() { @@ -21,25 +22,45 @@ func main() { app.Wait() } +// newLogger constructs a zap.Logger instance for current application. +// Panics on failure. +// +// Logger is built from zap's production logging configuration with: +// * parameterized level (debug by default) +// * console encoding +// * ISO8601 time encoding +// +// Logger records a stack trace for all messages at or above fatal level. +// +// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. func newLogger(v *viper.Viper) *zap.Logger { - options := []logger.Option{ - logger.WithLevel(v.GetString(cfgLoggerLevel)), - logger.WithTraceLevel(v.GetString(cfgLoggerTraceLevel)), - logger.WithFormat(v.GetString(cfgLoggerFormat)), - logger.WithSamplingInitial(v.GetInt(cfgLoggerSamplingInitial)), - logger.WithSamplingThereafter(v.GetInt(cfgLoggerSamplingThereafter)), - logger.WithAppName(v.GetString(cfgApplicationName)), - logger.WithAppVersion(v.GetString(cfgApplicationVersion)), - } - if v.GetBool(cfgLoggerNoCaller) { - options = append(options, logger.WithoutCaller()) - } - if v.GetBool(cfgLoggerNoDisclaimer) { - options = append(options, logger.WithoutDisclaimer()) - } - l, err := logger.New(options...) + var lvl zapcore.Level + lvlStr := v.GetString(cfgLoggerLevel) + err := lvl.UnmarshalText([]byte(lvlStr)) if err != nil { - panic(err) + panic(fmt.Sprintf("incorrect logger level configuration %s (%v), "+ + "value should be one of %v", lvlStr, err, [...]zapcore.Level{ + zapcore.DebugLevel, + zapcore.InfoLevel, + zapcore.WarnLevel, + zapcore.ErrorLevel, + zapcore.DPanicLevel, + zapcore.PanicLevel, + zapcore.FatalLevel, + })) } + + c := zap.NewProductionConfig() + c.Level = zap.NewAtomicLevelAt(lvl) + c.Encoding = "console" + c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + + l, err := c.Build( + zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), + ) + if err != nil { + panic(fmt.Sprintf("build zap logger instance: %v", err)) + } + return l } diff --git a/metrics.go b/metrics.go index b2f071e..6f8f6da 100644 --- a/metrics.go +++ b/metrics.go @@ -9,21 +9,14 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/expfmt" "github.com/valyala/fasthttp" + "go.uber.org/zap" ) -func attachMetrics(r *router.Router, z promhttp.Logger) { - r.GET("/metrics/", metricsHandler(prometheus.DefaultGatherer, promhttp.HandlerOpts{ - ErrorLog: z, - // ErrorHandling: 0, - // Registry: nil, - // DisableCompression: false, - // MaxRequestsInFlight: 0, - // Timeout: 0, - // EnableOpenMetrics: false, - })) +func attachMetrics(r *router.Router, l *zap.Logger) { + r.GET("/metrics/", metricsHandler(prometheus.DefaultGatherer, l, promhttp.HandlerOpts{})) } -func metricsHandler(reg prometheus.Gatherer, opts promhttp.HandlerOpts) fasthttp.RequestHandler { +func metricsHandler(reg prometheus.Gatherer, logger *zap.Logger, opts promhttp.HandlerOpts) fasthttp.RequestHandler { var ( inFlightSem chan struct{} errCnt = prometheus.NewCounterVec( @@ -66,7 +59,7 @@ func metricsHandler(reg prometheus.Gatherer, opts promhttp.HandlerOpts) fasthttp } mfs, err := reg.Gather() if err != nil { - if opts.ErrorLog != nil { + if logger != nil { panic("error gathering metrics:" + err.Error()) } @@ -99,8 +92,8 @@ func metricsHandler(reg prometheus.Gatherer, opts promhttp.HandlerOpts) fasthttp return false } lastErr = err - if opts.ErrorLog != nil { - opts.ErrorLog.Println("error encoding and sending metric family:", err) + if logger != nil { + logger.Error("encoding and sending metric family", zap.Error(err)) } errCnt.WithLabelValues("encoding").Inc() switch opts.ErrorHandling { diff --git a/settings.go b/settings.go index 5ab6323..841c1df 100644 --- a/settings.go +++ b/settings.go @@ -36,13 +36,7 @@ const ( cfgRebalance = "rebalance_timer" // Logger. - cfgLoggerLevel = "logger.level" - cfgLoggerFormat = "logger.format" - cfgLoggerTraceLevel = "logger.trace_level" - cfgLoggerNoCaller = "logger.no_caller" - cfgLoggerNoDisclaimer = "logger.no_disclaimer" - cfgLoggerSamplingInitial = "logger.sampling.initial" - cfgLoggerSamplingThereafter = "logger.sampling.thereafter" + cfgLoggerLevel = "logger.level" // Wallet. cfgWalletPassphrase = "wallet.passphrase" @@ -56,10 +50,6 @@ const ( // Zip compression. cfgZipCompression = "zip.compression" - // Application. - cfgApplicationName = "app.name" - cfgApplicationVersion = "app.version" - // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -72,11 +62,9 @@ const ( ) var ignore = map[string]struct{}{ - cfgApplicationName: {}, - cfgApplicationVersion: {}, - cfgPeers: {}, - cmdHelp: {}, - cmdVersion: {}, + cfgPeers: {}, + cmdHelp: {}, + cmdVersion: {}, } func settings() *viper.Viper { @@ -111,20 +99,10 @@ func settings() *viper.Viper { flags.String(cfgTLSKey, "", "TLS key path") peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") - // set prefers: - v.Set(cfgApplicationName, "neofs-http-gw") - v.Set(cfgApplicationVersion, Version) - // set defaults: // logger: v.SetDefault(cfgLoggerLevel, "debug") - v.SetDefault(cfgLoggerFormat, "console") - v.SetDefault(cfgLoggerTraceLevel, "panic") - v.SetDefault(cfgLoggerNoCaller, false) - v.SetDefault(cfgLoggerNoDisclaimer, true) - v.SetDefault(cfgLoggerSamplingInitial, 1000) - v.SetDefault(cfgLoggerSamplingThereafter, 1000) // web-server: v.SetDefault(cfgWebReadBufferSize, 4096) diff --git a/uploader/filter_test.go b/uploader/filter_test.go index 4dc8e0c..913f076 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -7,14 +7,13 @@ import ( "github.com/nspcc-dev/neofs-api-go/v2/object" "github.com/nspcc-dev/neofs-http-gw/utils" - "github.com/nspcc-dev/neofs-sdk-go/logger" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" + "go.uber.org/zap" ) func TestFilter(t *testing.T) { - log, err := logger.New() - require.NoError(t, err) + log := zap.NewNop() req := &fasthttp.RequestHeader{} req.DisableNormalizing() From 9475786df871a676ab93ae23f303ada085065a05 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 25 Mar 2022 16:12:26 +0300 Subject: [PATCH 248/548] [#137] Refactor unused metrics code Signed-off-by: Denis Kirillov --- metrics.go | 64 +++++++----------------------------------------------- 1 file changed, 8 insertions(+), 56 deletions(-) diff --git a/metrics.go b/metrics.go index 6f8f6da..40491bc 100644 --- a/metrics.go +++ b/metrics.go @@ -1,22 +1,19 @@ package main import ( - "fmt" - "github.com/fasthttp/router" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/expfmt" "github.com/valyala/fasthttp" "go.uber.org/zap" ) func attachMetrics(r *router.Router, l *zap.Logger) { - r.GET("/metrics/", metricsHandler(prometheus.DefaultGatherer, l, promhttp.HandlerOpts{})) + r.GET("/metrics/", metricsHandler(prometheus.DefaultGatherer, l)) } -func metricsHandler(reg prometheus.Gatherer, logger *zap.Logger, opts promhttp.HandlerOpts) fasthttp.RequestHandler { +func metricsHandler(reg prometheus.Gatherer, logger *zap.Logger) fasthttp.RequestHandler { var ( inFlightSem chan struct{} errCnt = prometheus.NewCounterVec( @@ -28,32 +25,13 @@ func metricsHandler(reg prometheus.Gatherer, logger *zap.Logger, opts promhttp.H ) ) - if opts.MaxRequestsInFlight > 0 { - inFlightSem = make(chan struct{}, opts.MaxRequestsInFlight) - } - if opts.Registry != nil { - // Initialize all possibilites that can occur below. - errCnt.WithLabelValues("gathering") - errCnt.WithLabelValues("encoding") - if err := opts.Registry.Register(errCnt); err != nil { - if are, ok := err.(prometheus.AlreadyRegisteredError); ok { - errCnt = are.ExistingCollector.(*prometheus.CounterVec) - } else { - panic(err) - } - } - } - h := fasthttp.RequestHandler(func(c *fasthttp.RequestCtx) { if inFlightSem != nil { select { case inFlightSem <- struct{}{}: // All good, carry on. defer func() { <-inFlightSem }() default: - - response.Error(c, fmt.Sprintf( - "Limit of concurrent requests reached (%d), try again later.", opts.MaxRequestsInFlight, - ), fasthttp.StatusServiceUnavailable) + response.Error(c, "Limit of concurrent requests reached, try again later.", fasthttp.StatusServiceUnavailable) return } } @@ -64,19 +42,8 @@ func metricsHandler(reg prometheus.Gatherer, logger *zap.Logger, opts promhttp.H } errCnt.WithLabelValues("gathering").Inc() - switch opts.ErrorHandling { - case promhttp.PanicOnError: - panic(err) - case promhttp.ContinueOnError: - if len(mfs) == 0 { - // Still report the error if no metrics have been gathered. - response.Error(c, err.Error(), fasthttp.StatusServiceUnavailable) - return - } - case promhttp.HTTPErrorOnError: - response.Error(c, err.Error(), fasthttp.StatusServiceUnavailable) - return - } + response.Error(c, err.Error(), fasthttp.StatusServiceUnavailable) + return } contentType := expfmt.FmtText @@ -96,16 +63,8 @@ func metricsHandler(reg prometheus.Gatherer, logger *zap.Logger, opts promhttp.H logger.Error("encoding and sending metric family", zap.Error(err)) } errCnt.WithLabelValues("encoding").Inc() - switch opts.ErrorHandling { - case promhttp.PanicOnError: - panic(err) - case promhttp.HTTPErrorOnError: - response.Error(c, err.Error(), fasthttp.StatusServiceUnavailable) - return true - default: - // Do nothing in all other cases, including ContinueOnError. - return false - } + response.Error(c, err.Error(), fasthttp.StatusServiceUnavailable) + return true } for _, mf := range mfs { @@ -123,12 +82,5 @@ func metricsHandler(reg prometheus.Gatherer, logger *zap.Logger, opts promhttp.H handleError(lastErr) }) - if opts.Timeout <= 0 { - return h - } - - return fasthttp.TimeoutHandler(h, opts.Timeout, fmt.Sprintf( - "Exceeded configured timeout of %v.\n", - opts.Timeout, - )) + return h } From 5080b43a04c46b7e846bd68463a7dc9cc925f27d Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 7 Apr 2022 15:56:18 +0300 Subject: [PATCH 249/548] [#139] Update SDK Signed-off-by: Denis Kirillov --- app.go | 31 ++++++++++++++++-------- downloader/download.go | 28 +++++++++++++++------ downloader/head.go | 16 +++++++++--- go.mod | 2 +- go.sum | 4 +-- integration_test.go | 42 ++++++++++++++++++++------------ uploader/upload.go | 55 +++++++++++++++++++++--------------------- 7 files changed, 111 insertions(+), 67 deletions(-) diff --git a/app.go b/app.go index 63334cd..487d356 100644 --- a/app.go +++ b/app.go @@ -9,6 +9,7 @@ import ( "github.com/fasthttp/router" "github.com/nspcc-dev/neo-go/cli/flags" "github.com/nspcc-dev/neo-go/cli/input" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/nspcc-dev/neofs-http-gw/downloader" @@ -92,7 +93,13 @@ func newApp(ctx context.Context, opt ...Option) App { if err != nil { a.log.Fatal("failed to get neofs credentials", zap.Error(err)) } - pb := new(pool.Builder) + + var prm pool.InitParameters + prm.SetKey(key) + prm.SetNodeDialTimeout(a.cfg.GetDuration(cfgConTimeout)) + prm.SetHealthcheckTimeout(a.cfg.GetDuration(cfgReqTimeout)) + prm.SetClientRebalanceInterval(a.cfg.GetDuration(cfgRebalance)) + for i := 0; ; i++ { address := a.cfg.GetString(cfgPeers + "." + strconv.Itoa(i) + ".address") weight := a.cfg.GetFloat64(cfgPeers + "." + strconv.Itoa(i) + ".weight") @@ -106,20 +113,20 @@ func newApp(ctx context.Context, opt ...Option) App { if priority <= 0 { // unspecified or wrong priority = 1 } - pb.AddNode(address, priority, weight) + prm.AddNode(pool.NewNodeParam(priority, address, weight)) a.log.Info("add connection", zap.String("address", address), zap.Float64("weight", weight), zap.Int("priority", priority)) } - opts := &pool.BuilderOptions{ - Key: key, - NodeConnectionTimeout: a.cfg.GetDuration(cfgConTimeout), - NodeRequestTimeout: a.cfg.GetDuration(cfgReqTimeout), - ClientRebalanceInterval: a.cfg.GetDuration(cfgRebalance), - } - a.pool, err = pb.Build(ctx, opts) + + a.pool, err = pool.NewPool(prm) if err != nil { a.log.Fatal("failed to create connection pool", zap.Error(err)) } + + err = a.pool.Dial(ctx) + if err != nil { + a.log.Fatal("failed to dial pool", zap.Error(err)) + } return a } @@ -127,7 +134,11 @@ func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) { walletPath := a.cfg.GetString(cmdWallet) if len(walletPath) == 0 { a.log.Info("no wallet path specified, creating ephemeral key automatically for this run") - return pool.NewEphemeralKey() + key, err := keys.NewPrivateKey() + if err != nil { + return nil, err + } + return &key.PrivateKey, nil } w, err := wallet.NewWalletFromFile(walletPath) if err != nil { diff --git a/downloader/download.go b/downloader/download.go index 3599119..3221c45 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -22,6 +22,7 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/object/address" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/pool" + "github.com/nspcc-dev/neofs-sdk-go/token" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -97,7 +98,11 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress *address.Address) { return } - rObj, err := clnt.GetObject(r.RequestCtx, *objectAddress, bearerOpts(r.RequestCtx)) + var prm pool.PrmObjectGet + prm.SetAddress(*objectAddress) + prm.UseBearer(bearerToken(r.RequestCtx)) + + rObj, err := clnt.GetObject(r.RequestCtx, prm) if err != nil { r.handleNeoFSErr(err, start) return @@ -195,11 +200,11 @@ func systemBackwardTranslator(key string) string { return res.String() } -func bearerOpts(ctx context.Context) pool.CallOption { +func bearerToken(ctx context.Context) *token.BearerToken { if tkn, err := tokens.LoadBearerToken(ctx); err == nil { - return pool.WithBearer(tkn) + return tkn } - return pool.WithBearer(nil) + return nil } func (r *request) handleNeoFSErr(err error, start time.Time) { @@ -335,7 +340,12 @@ func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string filters.AddRootFilter() filters.AddFilter(key, val, op) - return d.pool.SearchObjects(c, *cid, filters) + var prm pool.PrmObjectSearch + prm.SetContainerID(*cid) + prm.SetFilters(filters) + prm.UseBearer(bearerToken(c)) + + return d.pool.SearchObjects(c, prm) } // DownloadZipped handles zip by prefix requests. @@ -385,7 +395,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { addr.SetContainerID(containerID) - optBearer := bearerOpts(c) + btoken := bearerToken(c) empty := true called := false @@ -400,7 +410,11 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { addr.SetObjectID(&id) - resGet, err = d.pool.GetObject(c, addr, optBearer) + var prm pool.PrmObjectGet + prm.SetAddress(addr) + prm.UseBearer(btoken) + + resGet, err = d.pool.GetObject(c, prm) if err != nil { err = fmt.Errorf("get NeoFS object: %v", err) return true diff --git a/downloader/head.go b/downloader/head.go index ddcc770..a299d31 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -33,8 +33,13 @@ func (r request) headObject(clnt *pool.Pool, objectAddress *address.Address) { return } - bearerOpt := bearerOpts(r.RequestCtx) - obj, err := clnt.HeadObject(r.RequestCtx, *objectAddress, bearerOpt) + btoken := bearerToken(r.RequestCtx) + + var prm pool.PrmObjectHead + prm.SetAddress(*objectAddress) + prm.UseBearer(btoken) + + obj, err := clnt.HeadObject(r.RequestCtx, prm) if err != nil { r.handleNeoFSErr(err, start) return @@ -69,7 +74,12 @@ func (r request) headObject(clnt *pool.Pool, objectAddress *address.Address) { if len(contentType) == 0 { contentType, _, err = readContentType(obj.PayloadSize(), func(sz uint64) (io.Reader, error) { - return clnt.ObjectRange(r.RequestCtx, *objectAddress, 0, sz, bearerOpt) + var prmRange pool.PrmObjectRange + prmRange.SetAddress(*objectAddress) + prmRange.SetLength(sz) + prmRange.UseBearer(btoken) + + return clnt.ObjectRange(r.RequestCtx, prmRange) }) if err != nil && err != io.EOF { r.handleNeoFSErr(err, start) diff --git a/go.mod b/go.mod index 046ec64..bf8c871 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/nspcc-dev/neo-go v0.98.0 github.com/nspcc-dev/neofs-api-go/v2 v2.12.1 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220407103316-e50e6d28280d github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 github.com/prometheus/procfs v0.7.1 // indirect diff --git a/go.sum b/go.sum index ab2b770..70dc092 100644 --- a/go.sum +++ b/go.sum @@ -596,8 +596,8 @@ github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BE github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3 h1:ofaiKPYY67a0cQMF+YSChDO48SBQtWlpZnK++cAeqQM= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3/go.mod h1:0hTXmyJnbw8j4BR1oltN7mFIIrVp1IFLdh8qBzAR464= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220407103316-e50e6d28280d h1:OHyq8+zyQtARFWj3quRPabcfQWJZEiU7HYp6QGCSjaM= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220407103316-e50e6d28280d/go.mod h1:Hl7a1l0ntZ4b1ZABpGX6fuAuFS3c6+hyMCUNVvZv/w4= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/integration_test.go b/integration_test.go index 882c138..e1d15ae 100644 --- a/integration_test.go +++ b/integration_test.go @@ -119,7 +119,10 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci payload := bytes.NewBuffer(nil) - res, err := clientPool.GetObject(ctx, *objectAddress) + var prm pool.PrmObjectGet + prm.SetAddress(*objectAddress) + + res, err := clientPool.GetObject(ctx, prm) require.NoError(t, err) _, err = io.Copy(payload, res.Payload) @@ -272,15 +275,15 @@ func getDefaultConfig() *viper.Viper { } func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool { - pb := new(pool.Builder) - pb.AddNode("localhost:8080", 1, 1) + var prm pool.InitParameters + prm.SetKey(&key.PrivateKey) + prm.SetNodeDialTimeout(5 * time.Second) + prm.AddNode(pool.NewNodeParam(1, "localhost:8080", 1)) - opts := &pool.BuilderOptions{ - Key: &key.PrivateKey, - NodeConnectionTimeout: 5 * time.Second, - NodeRequestTimeout: 5 * time.Second, - } - clientPool, err := pb.Build(ctx, opts) + clientPool, err := pool.NewPool(prm) + require.NoError(t, err) + + err = clientPool.Dial(ctx) require.NoError(t, err) return clientPool } @@ -296,17 +299,20 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool) ( container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10))) cnr.SetOwnerID(clientPool.OwnerID()) - CID, err := clientPool.PutContainer(ctx, cnr) + var waitPrm pool.WaitParams + waitPrm.SetTimeout(15 * time.Second) + waitPrm.SetPollInterval(3 * time.Second) + + var prm pool.PrmContainerPut + prm.SetContainer(*cnr) + prm.SetWaitParams(waitPrm) + + CID, err := clientPool.PutContainer(ctx, prm) if err != nil { return nil, err } fmt.Println(CID.String()) - err = clientPool.WaitForContainerPresence(ctx, CID, &pool.ContainerPollingParams{ - CreationTimeout: 15 * time.Second, - PollInterval: 3 * time.Second, - }) - return CID, err } @@ -324,7 +330,11 @@ func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci } obj.SetAttributes(attrs...) - id, err := clientPool.PutObject(ctx, *obj, bytes.NewBufferString(content)) + var prm pool.PrmObjectPut + prm.SetHeader(*obj) + prm.SetPayload(bytes.NewBufferString(content)) + + id, err := clientPool.PutObject(ctx, prm) require.NoError(t, err) return id diff --git a/uploader/upload.go b/uploader/upload.go index 4b094a5..7b0798b 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -12,8 +12,6 @@ import ( "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" - "github.com/nspcc-dev/neofs-sdk-go/client" - apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" @@ -139,7 +137,12 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { ctx, cancel := context.WithCancel(c) defer cancel() - if idObj, err = u.pool.PutObject(ctx, *obj, file, pool.WithBearer(bt)); err != nil { + var prm pool.PrmObjectPut + prm.SetHeader(*obj) + prm.SetPayload(file) + prm.UseBearer(bt) + + if idObj, err = u.pool.PutObject(ctx, prm); err != nil { log.Error("could not store file in neofs", zap.Error(err)) response.Error(c, "could not store file in neofs", fasthttp.StatusBadRequest) return @@ -198,33 +201,29 @@ func (pr *putResponse) encode(w io.Writer) error { } func getEpochDurations(ctx context.Context, p *pool.Pool) (*epochDurations, error) { - if conn, _, err := p.Connection(); err != nil { + networkInfo, err := p.NetworkInfo(ctx) + if err != nil { return nil, err - } else if networkInfoRes, err := conn.NetworkInfo(ctx, client.PrmNetworkInfo{}); err != nil { - return nil, err - } else if err = apistatus.ErrFromStatus(networkInfoRes.Status()); err != nil { - return nil, err - } else { - networkInfo := networkInfoRes.Info() - res := &epochDurations{ - currentEpoch: networkInfo.CurrentEpoch(), - msPerBlock: networkInfo.MsPerBlock(), - } - - networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool { - if string(parameter.Key()) == "EpochDuration" { - data := make([]byte, 8) - copy(data, parameter.Value()) - res.blockPerEpoch = binary.LittleEndian.Uint64(data) - return true - } - return false - }) - if res.blockPerEpoch == 0 { - return nil, fmt.Errorf("not found param: EpochDuration") - } - return res, nil } + + res := &epochDurations{ + currentEpoch: networkInfo.CurrentEpoch(), + msPerBlock: networkInfo.MsPerBlock(), + } + + networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool { + if string(parameter.Key()) == "EpochDuration" { + data := make([]byte, 8) + copy(data, parameter.Value()) + res.blockPerEpoch = binary.LittleEndian.Uint64(data) + return true + } + return false + }) + if res.blockPerEpoch == 0 { + return nil, fmt.Errorf("not found param: EpochDuration") + } + return res, nil } func needParseExpiration(headers map[string]string) bool { From baf425453d7ccf1ed61a7937768dafe846acc6e4 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 7 Apr 2022 16:43:59 +0300 Subject: [PATCH 250/548] [#139] fix lint Signed-off-by: Denis Kirillov --- app.go | 6 ++---- downloader/download.go | 23 +++++++++++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/app.go b/app.go index 487d356..22aaadb 100644 --- a/app.go +++ b/app.go @@ -199,10 +199,7 @@ func (a *app) Serve(ctx context.Context) { }() edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) uploader := uploader.New(a.log, a.pool, edts) - downloader, err := downloader.New(a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool) - if err != nil { - a.log.Fatal("failed to create downloader", zap.Error(err)) - } + downloader := downloader.New(a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool) // Configure router. r := router.New() r.RedirectTrailingSlash = true @@ -237,6 +234,7 @@ func (a *app) Serve(ctx context.Context) { tlsKeyPath := a.cfg.GetString(cfgTLSKey) a.webServer.Handler = r.Handler + var err error if tlsCertPath == "" && tlsKeyPath == "" { a.log.Info("running web server", zap.String("address", bind)) err = a.webServer.ListenAndServe(bind) diff --git a/downloader/download.go b/downloader/download.go index 3221c45..03fce82 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -13,6 +13,8 @@ import ( "strconv" "strings" "time" + "unicode" + "unicode/utf8" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" @@ -190,7 +192,7 @@ func systemBackwardTranslator(key string) string { strs := strings.Split(key, "_") for i, s := range strs { - s = strings.Title(strings.ToLower(s)) + s = title(strings.ToLower(s)) res.WriteString(s) if i != len(strs)-1 { res.WriteString("-") @@ -200,6 +202,16 @@ func systemBackwardTranslator(key string) string { return res.String() } +func title(str string) string { + if str == "" { + return "" + } + + r, size := utf8.DecodeRuneInString(str) + r0 := unicode.ToTitle(r) + return string(r0) + str[size:] +} + func bearerToken(ctx context.Context) *token.BearerToken { if tkn, err := tokens.LoadBearerToken(ctx); err == nil { return tkn @@ -243,13 +255,8 @@ type Settings struct { } // New creates an instance of Downloader using specified options. -func New(log *zap.Logger, settings Settings, conns *pool.Pool) (*Downloader, error) { - var err error - d := &Downloader{log: log, pool: conns, settings: settings} - if err != nil { - return nil, fmt.Errorf("failed to get neofs client's reusable artifacts: %w", err) - } - return d, nil +func New(log *zap.Logger, settings Settings, conns *pool.Pool) *Downloader { + return &Downloader{log: log, pool: conns, settings: settings} } func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { From eacd7be33e004822e9ace676a1ada265daf79392 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 8 Apr 2022 12:01:57 +0300 Subject: [PATCH 251/548] [#140] Improve docs Signed-off-by: Denis Kirillov --- README.md | 49 +++------------------------- app.go | 12 ++++++- config/config.env | 80 ++++++++++++++++++++++++++++++++++++++++++++++ config/config.yaml | 78 ++++++++++++++++++++++++++++++++++++++++++++ settings.go | 4 +-- 5 files changed, 175 insertions(+), 48 deletions(-) create mode 100644 config/config.env create mode 100644 config/config.yaml diff --git a/README.md b/README.md index 866e715..9bea669 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ $ HTTP_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 neofs-http-gw ## Configuration In general, everything available as CLI parameter can also be specified via -environment variables, so they're not specifically mentioned in most cases +environment variables (see [example](./config/config.env)), so they're not specifically mentioned in most cases (see `--help` also). If you prefer a config file you can use it in yaml format. ### Nodes: weights and priorities @@ -162,18 +162,9 @@ The gateway supports downloading files by common prefix (like dir) in zip format using config or `HTTP_GW_ZIP_COMPRESSION=true` environment variable. ### Logging - -`--verbose` flag enables gRPC logging and there is a number of environment -variables to tune logging behavior: - +You can specify logging level (default `info`) using variable: ``` -HTTP_GW_LOGGER_FORMAT=string - Logger format -HTTP_GW_LOGGER_LEVEL=string - Logger level -HTTP_GW_LOGGER_NO_CALLER=bool - Logger don't show caller -HTTP_GW_LOGGER_NO_DISCLAIMER=bool - Logger don't show application name/version -HTTP_GW_LOGGER_SAMPLING_INITIAL=int - Logger sampling initial -HTTP_GW_LOGGER_SAMPLING_THEREAFTER=int - Logger sampling thereafter -HTTP_GW_LOGGER_TRACE_LEVEL=string - Logger show trace on level +HTTP_GW_LOGGER_LEVEL=debug ``` ### Yaml file @@ -183,39 +174,7 @@ It can be specified with `--config` parameter: $ neofs-http-gw --config your-config.yaml ``` -Configuration file example: -``` -listen_address: 0.0.0.0:8082 - -wallet: - passphrase: 123456 - -logger: - level: debug - -peers: - 0: - address: grpc://s01.neofs.devenv:8080 - weight: 1 - priority: 1 - -zip: - compression: false -``` - -To know nesting level of variable you need to cut off the prefix `HTTP_GW` from variable and split the rest parts by `_`. -For example variable `HTTP_GW_PEERS_0_WEIGHT=1` will be transformed to: -``` -peers: - 0: - weight: 1 - priority: 1 -``` - -If parameter doesn't support environment variable (e.g. `--listen_address 0.0.0.0:8082`) form it is used as is: -``` -listen_address: 0.0.0.0:8082 -``` +See [config](./config/config.yaml) for example. ## HTTP API provided diff --git a/app.go b/app.go index 22aaadb..9e669eb 100644 --- a/app.go +++ b/app.go @@ -132,6 +132,10 @@ func newApp(ctx context.Context, opt ...Option) App { func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) { walletPath := a.cfg.GetString(cmdWallet) + if len(walletPath) == 0 { + walletPath = a.cfg.GetString(cfgWalletPath) + } + if len(walletPath) == 0 { a.log.Info("no wallet path specified, creating ephemeral key automatically for this run") key, err := keys.NewPrivateKey() @@ -150,7 +154,13 @@ func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) { pwd := a.cfg.GetString(cfgWalletPassphrase) password = &pwd } - return getKeyFromWallet(w, a.cfg.GetString(cmdAddress), password) + + address := a.cfg.GetString(cmdAddress) + if len(address) == 0 { + address = a.cfg.GetString(cfgWalletAddress) + } + + return getKeyFromWallet(w, address, password) } func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*ecdsa.PrivateKey, error) { diff --git a/config/config.env b/config/config.env new file mode 100644 index 0000000..006d94e --- /dev/null +++ b/config/config.env @@ -0,0 +1,80 @@ +# Wallet section. + +# Path to wallet. +HTTP_GW_WALLET_PATH=/path/to/wallet.json +# Account address. If omitted default one will be used. +HTTP_GW_WALLET_ADDRESS=NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP +# Password to decrypt wallet. +HTTP_GW_WALLET_PASSPHRASE=pwd + +# Enable metrics. +HTTP_GW_METRICS=true +# Enable pprof. +HTTP_GW_PPROF=true +# Log level. +HTTP_GW_LOGGER_LEVEL=debug + +# Address to bind. +HTTP_GW_LISTEN_ADDRESS=0.0.0.0:443 +# Provide cert to enable TLS. +HTTP_GW_TLS_CERTIFICATE=/path/to/tls/cert +# Provide key to enable TLS. +HTTP_GW_TLS_KEY=/path/to/tls/key + +# Nodes configuration. +# This configuration make gateway use the first node (grpc://s01.neofs.devenv:8080) +# while it's healthy. Otherwise, gateway use the second node (grpc://s01.neofs.devenv:8080) +# for 10% of requests and the third node for 90% of requests. + +# Peer 1. +# Endpoint. +HTTP_GW_PEERS_0_ADDRESS=grpc://s01.neofs.devenv:8080 +# Until nodes with the same priority level are healthy +# nodes with other priority are not used. +# Еhe lower the value, the higher the priority. +HTTP_GW_PEERS_0_PRIORITY=1 +# Load distribution proportion for nodes with the same priority. +HTTP_GW_PEERS_0_WEIGHT=1 +# Peer 2. +HTTP_GW_PEERS_1_ADDRESS=grpc://s02.neofs.devenv:8080 +HTTP_GW_PEERS_1_PRIORITY=2 +HTTP_GW_PEERS_1_WEIGHT=1 +# Peer 3. +HTTP_GW_PEERS_2_ADDRESS=grpc://s03.neofs.devenv:8080 +HTTP_GW_PEERS_2_PRIORITY=2 +HTTP_GW_PEERS_2_WEIGHT=9 + +# Per-connection buffer size for requests' reading. +# This also limits the maximum header size. +HTTP_GW_WEB_READ_BUFFER_SIZE=4096 +# Per-connection buffer size for responses' writing. +HTTP_GW_WRITE_BUFFER_SIZE=4096 +# ReadTimeout is the amount of time allowed to read +# the full request including body. The connection's read +# deadline is reset when the connection opens, or for +# keep-alive connections after the first byte has been read. +HTTP_GW_READ_TIMEOUT=15s +# WriteTimeout is the maximum duration before timing out +# writes of the response. It is reset after the request handler +# has returned. +HTTP_GW_WRITE_TIMEOUT=1m +# StreamRequestBody enables request body streaming, +# and calls the handler sooner when given body is +# larger then the current limit. +HTTP_GW_STREAM_REQUEST_BODY=true +# Maximum request body size. +# The server rejects requests with bodies exceeding this limit. +HTTP_GW_MAX_REQUEST_BODY_SIZE=4194304 + +# Create timestamp for object if it isn't provided by header. +HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=false + +# Timeout to dial node. +HTTP_GW_CONNECT_TIMEOUT=5s +# Timeout to check node health during rebalance. +HTTP_GW_REQUEST_TIMEOUT=5s +# Interval to check nodes health. +HTTP_GW_REBALANCE_TIMER=30s + +# Enable zip compression to download files by common prefix. +HTTP_GW_ZIP_COMPRESSION=false diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..9a2a50e --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,78 @@ +wallet: + path: /path/to/wallet.json # Path to wallet. + address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP # Account address. If omitted default one will be used. + passphrase: pwd # Password to decrypt wallet. + +metrics: true # Enable metrics. +pprof: true # Enable pprof. +logger: + level: debug # Log level. + +listen_address: 0.0.0.0:443 # Address to bind. +tls_certificate: /path/to/tls/cert # Provide cert to enable TLS. +tls_key: /path/to/tls/key # Provide key to enable TLS. + +# Nodes configuration. +# This configuration make gateway use the first node (grpc://s01.neofs.devenv:8080) +# while it's healthy. Otherwise, gateway use the second node (grpc://s01.neofs.devenv:8080) +# for 10% of requests and the third node for 90% of requests. +peers: + 0: + # Endpoint. + address: grpc://s01.neofs.devenv:8080 + + # Until nodes with the same priority level are healthy + # nodes with other priority are not used. + # Еhe lower the value, the higher the priority. + priority: 1 + + # Load distribution proportion for nodes with the same priority. + weight: 1 + 1: + address: grpc://s02.neofs.devenv:8080 + priority: 2 + weight: 1 + 2: + address: grpc://s03.neofs.devenv:8080 + priority: 2 + weight: 9 + + +web: + # Per-connection buffer size for requests' reading. + # This also limits the maximum header size. + read_buffer_size: 4096 + + # Per-connection buffer size for responses' writing. + write_buffer_size: 4096 + + # ReadTimeout is the amount of time allowed to read + # the full request including body. The connection's read + # deadline is reset when the connection opens, or for + # keep-alive connections after the first byte has been read. + read_timeout: 15s + + # WriteTimeout is the maximum duration before timing out + # writes of the response. It is reset after the request handler + # has returned. + write_timeout: 1m + + # StreamRequestBody enables request body streaming, + # and calls the handler sooner when given body is + # larger then the current limit. + stream_request_body: true + + # Maximum request body size. + # The server rejects requests with bodies exceeding this limit. + max_request_body_size: 4194304 + + +upload_header: + use_default_timestamp: false # Create timestamp for object if it isn't provided by header. + +connect_timeout: 5s # Timeout to dial node. +request_timeout: 5s # Timeout to check node health during rebalance. +rebalance_timer: 30s # Interval to check nodes health. + +zip: + compression: false # Enable zip compression to download files by common prefix. diff --git a/settings.go b/settings.go index 841c1df..624a786 100644 --- a/settings.go +++ b/settings.go @@ -40,6 +40,8 @@ const ( // Wallet. cfgWalletPassphrase = "wallet.passphrase" + cfgWalletPath = "wallet.path" + cfgWalletAddress = "wallet.address" // Uploader Header. cfgUploaderHeaderEnableDefaultTimestamp = "upload_header.use_default_timestamp" @@ -53,7 +55,6 @@ const ( // Command line args. cmdHelp = "help" cmdVersion = "version" - cmdVerbose = "verbose" cmdPprof = "pprof" cmdMetrics = "metrics" cmdWallet = "wallet" @@ -89,7 +90,6 @@ func settings() *viper.Viper { flags.StringP(cmdWallet, "w", "", `path to the wallet`) flags.String(cmdAddress, "", `address of wallet account`) config := flags.String(cmdConfig, "", "config path") - flags.Bool(cmdVerbose, false, "debug gRPC connections") flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout") flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout") flags.Duration(cfgRebalance, defaultRebalanceTimer, "gRPC connection rebalance timer") From 48ce84e65f94a5c238311bd276416a5b1fb57651 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 12 Apr 2022 09:20:18 +0300 Subject: [PATCH 252/548] [#140] Fix typos Signed-off-by: Denis Kirillov --- config/config.env | 8 ++++---- config/config.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/config/config.env b/config/config.env index 006d94e..fffa851 100644 --- a/config/config.env +++ b/config/config.env @@ -4,7 +4,7 @@ HTTP_GW_WALLET_PATH=/path/to/wallet.json # Account address. If omitted default one will be used. HTTP_GW_WALLET_ADDRESS=NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP -# Password to decrypt wallet. +# Passphrase to decrypt wallet. HTTP_GW_WALLET_PASSPHRASE=pwd # Enable metrics. @@ -22,8 +22,8 @@ HTTP_GW_TLS_CERTIFICATE=/path/to/tls/cert HTTP_GW_TLS_KEY=/path/to/tls/key # Nodes configuration. -# This configuration make gateway use the first node (grpc://s01.neofs.devenv:8080) -# while it's healthy. Otherwise, gateway use the second node (grpc://s01.neofs.devenv:8080) +# This configuration make the gateway use the first node (grpc://s01.neofs.devenv:8080) +# while it's healthy. Otherwise, the gateway use the second node (grpc://s01.neofs.devenv:8080) # for 10% of requests and the third node for 90% of requests. # Peer 1. @@ -31,7 +31,7 @@ HTTP_GW_TLS_KEY=/path/to/tls/key HTTP_GW_PEERS_0_ADDRESS=grpc://s01.neofs.devenv:8080 # Until nodes with the same priority level are healthy # nodes with other priority are not used. -# Еhe lower the value, the higher the priority. +# The lower the value, the higher the priority. HTTP_GW_PEERS_0_PRIORITY=1 # Load distribution proportion for nodes with the same priority. HTTP_GW_PEERS_0_WEIGHT=1 diff --git a/config/config.yaml b/config/config.yaml index 9a2a50e..26c0d30 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,7 +1,7 @@ wallet: path: /path/to/wallet.json # Path to wallet. address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP # Account address. If omitted default one will be used. - passphrase: pwd # Password to decrypt wallet. + passphrase: pwd # Passphrase to decrypt wallet. metrics: true # Enable metrics. pprof: true # Enable pprof. @@ -13,8 +13,8 @@ tls_certificate: /path/to/tls/cert # Provide cert to enable TLS. tls_key: /path/to/tls/key # Provide key to enable TLS. # Nodes configuration. -# This configuration make gateway use the first node (grpc://s01.neofs.devenv:8080) -# while it's healthy. Otherwise, gateway use the second node (grpc://s01.neofs.devenv:8080) +# This configuration make the gateway use the first node (grpc://s01.neofs.devenv:8080) +# while it's healthy. Otherwise, the gateway use the second node (grpc://s01.neofs.devenv:8080) # for 10% of requests and the third node for 90% of requests. peers: 0: @@ -23,7 +23,7 @@ peers: # Until nodes with the same priority level are healthy # nodes with other priority are not used. - # Еhe lower the value, the higher the priority. + # The lower the value, the higher the priority. priority: 1 # Load distribution proportion for nodes with the same priority. From 1c2fec81823f9b363f785b84705d8d4ee70b3ade Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 19 Apr 2022 15:02:18 +0300 Subject: [PATCH 253/548] [#147] Download zip by FilePath attribute Signed-off-by: Denis Kirillov --- downloader/download.go | 10 ++++++---- integration_test.go | 6 ++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 03fce82..554987f 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -36,6 +36,8 @@ type request struct { var errObjectNotFound = errors.New("object not found") +const attributeFilePath = "FilePath" + func isValidToken(s string) bool { for _, c := range s { if c <= ' ' || c > 127 { @@ -374,7 +376,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { return } - resSearch, err := d.search(c, containerID, object.AttributeFileName, prefix, object.MatchCommonPrefix) + resSearch, err := d.search(c, containerID, attributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error("could not search for objects", zap.Error(err)) response.Error(c, "could not search for objects", fasthttp.StatusBadRequest) @@ -428,7 +430,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { } w, err = zipWriter.CreateHeader(&zip.FileHeader{ - Name: getFilename(&resGet.Header), + Name: getZipFilePath(&resGet.Header), Method: compression, Modified: time.Now(), }) @@ -474,9 +476,9 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { } } -func getFilename(obj *object.Object) string { +func getZipFilePath(obj *object.Object) string { for _, attr := range obj.Attributes() { - if attr.Key() == object.AttributeFileName { + if attr.Key() == attributeFilePath { return attr.Value() } } diff --git a/integration_test.go b/integration_test.go index e1d15ae..174db56 100644 --- a/integration_test.go +++ b/integration_test.go @@ -28,6 +28,8 @@ import ( "github.com/testcontainers/testcontainers-go/wait" ) +const attributeFilePath = "FilePath" + type putResponse struct { CID string `json:"container_id"` OID string `json:"object_id"` @@ -191,8 +193,8 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"} contents := []string{"content of file1", "content of file2"} - attributes1 := map[string]string{object.AttributeFileName: names[0]} - attributes2 := map[string]string{object.AttributeFileName: names[1]} + attributes1 := map[string]string{attributeFilePath: names[0]} + attributes2 := map[string]string{attributeFilePath: names[1]} putObject(ctx, t, clientPool, CID, contents[0], attributes1) putObject(ctx, t, clientPool, CID, contents[1], attributes2) From 0b57b7add3bd59659d99862011d64ca56c1fe740 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 19 Apr 2022 15:09:00 +0300 Subject: [PATCH 254/548] [#147] Update README Signed-off-by: Denis Kirillov --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 9bea669..8126473 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,14 @@ You can download some dir (files with the same prefix) in zip (it will be compre $ wget http://localhost:8082/zip/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/common/prefix ``` +**Note:** the objects must have a `FilePath` attribute, otherwise they will not be in the zip archive. +You can upload file with this attribute using `curl`: + +``` +$ curl -F 'file=@cat.jpeg;filename=cat.jpeg' -H 'X-Attribute-FilePath: common/prefix/cat.jpeg' http://localhost:8082/upload/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ +``` + + #### Replies You get object contents in the reply body (if GET method was used), but at the same time you also get a From 54151992abbe5ea7d11684e7e1bd7af333011b84 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 18 Apr 2022 13:10:13 +0300 Subject: [PATCH 255/548] [#143] Update testcontainers-go to v0.13.0 This fixes vulnerabilities in indirect dependencies: - docker/distribution (CWE-843) - opencontainers/runc (CWE-190) - opencontainers/image-spec (CWE-843) Signed-off-by: Alex Vanin --- go.mod | 20 +-- go.sum | 386 +++++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 358 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index bf8c871..487e2ff 100644 --- a/go.mod +++ b/go.mod @@ -3,26 +3,32 @@ module github.com/nspcc-dev/neofs-http-gw go 1.16 require ( + github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/containerd/containerd v1.6.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/docker v20.10.14+incompatible // indirect github.com/fasthttp/router v1.4.1 - github.com/golang/mock v1.6.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + github.com/magiconair/properties v1.8.6 // indirect + github.com/moby/sys/mount v0.3.2 // indirect github.com/nspcc-dev/neo-go v0.98.0 github.com/nspcc-dev/neofs-api-go/v2 v2.12.1 github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220407103316-e50e6d28280d + github.com/opencontainers/runc v1.1.1 // indirect github.com/prometheus/client_golang v1.11.0 - github.com/prometheus/common v0.29.0 - github.com/prometheus/procfs v0.7.1 // indirect + github.com/prometheus/common v0.30.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect - github.com/testcontainers/testcontainers-go v0.12.0 + github.com/testcontainers/testcontainers-go v0.13.0 github.com/valyala/fasthttp v1.34.0 go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.18.1 - golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect - golang.org/x/tools v0.1.5 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect + golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect + google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect ) diff --git a/go.sum b/go.sum index 70dc092..818ae84 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,5 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -38,18 +39,24 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -57,27 +64,37 @@ github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1 github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= -github.com/Flaque/filet v0.0.0-20201012163910-45f684403088 h1:PnnQln5IGbhLeJOi6hVs+lCeF+B1dRfFKPGXUAez0Ww= -github.com/Flaque/filet v0.0.0-20201012163910-45f684403088/go.mod h1:TK+jB3mBs+8ZMWhU5BqZKnZWJ1MrLo8etNVg51ueTBo= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3 h1:mw6pDQqv38/WGF1cO/jF5t/jyAJ2yi7CmtFLLO5tGFI= github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= -github.com/Microsoft/hcsshim v0.8.16 h1:8/auA4LFIZFTGrqfKhGBSXwM6/4X1fHa/xniyEHu8ac= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= +github.com/Microsoft/hcsshim v0.9.2 h1:wB06W5aYFfUB3IvootYAY2WnOmIdgPGfqSI6tufQNnY= +github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= @@ -91,6 +108,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= @@ -100,10 +118,12 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1 h1:zFRi26YWd7NIorBXe8UkevRl0dIvk/AnXHWaAiZG+Yk= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -114,6 +134,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -133,47 +154,70 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 h1:hkGVFjz+plgr5UfxZUTPFbUFIF/Km6/s+RVRIRHLrrY= github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBdAP4= +github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -182,49 +226,82 @@ github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= -github.com/containerd/containerd v1.5.0-beta.4 h1:zjz4MOAOFgdBlwid2nNUlJ3YLpVi/97L36lfMYJex60= github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= +github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= +github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= +github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= +github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= +github.com/containerd/containerd v1.6.2 h1:pcaPUGbYW8kBw6OgIZwIVIeEhdWVrBzsoCfVJ5BjrLU= +github.com/containerd/containerd v1.6.2/go.mod h1:sidY30/InSE1j2vdD1ihtKoJz+lWdaXMdiAeIupaf+s= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= -github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e h1:6JKvHHt396/qabvMhnhUZvWaHZzfVfldxE60TK8YLhg= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= +github.com/containerd/continuity v0.2.2 h1:QSqfxcn8c+12slxwu00AtzXrsami0MJb/MQs9lOLHLA= +github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= +github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= +github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= +github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= +github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= +github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= +github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -245,6 +322,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= @@ -257,13 +335,20 @@ github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.11+incompatible h1:OqzI/g/W54LczvhnccGqniFoQghHx3pklbLuhfXpqGo= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.14+incompatible h1:+T9/PRYWNDo5SZl5qS1r9Mo/0Q8AwxKKPtu9S1yxM0w= +github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= @@ -286,16 +371,21 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fasthttp/router v1.4.1 h1:3xPUO+hy/HAkgGDSd5sX5w18cyGDIFbC7vip8KwPDk8= github.com/fasthttp/router v1.4.1/go.mod h1:4P0Kq4C882tA2evBKDW7De7hGfWmvV8FN+zqt8Lu49Q= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -303,6 +393,7 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -317,25 +408,40 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -347,10 +453,12 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -387,6 +495,7 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -400,8 +509,10 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -417,6 +528,7 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -426,28 +538,37 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -473,19 +594,26 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -498,6 +626,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -516,11 +645,17 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= +github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -534,14 +669,18 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= @@ -550,22 +689,31 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= -github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= +github.com/moby/sys/mount v0.3.2 h1:uq/CiGDZPvr+c85RYHtKIUORFbmavBUyWH3E1NEyjqI= +github.com/moby/sys/mount v0.3.2/go.mod h1:iN27Ec0LtJ0Mx/++rE6t6mTdbbEEZd+oKfAHP1y6vHs= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9ObI= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/mountinfo v0.6.1 h1:+H/KnGEAGRpTrEAqNVQ2AM3SiwMgJUt/TXj+Z8cmCIc= +github.com/moby/sys/mountinfo v0.6.1/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -576,6 +724,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= @@ -601,28 +750,38 @@ github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220407103316-e50e6d28280d/go.m github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -630,15 +789,19 @@ github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go. github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= -github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.1 h1:PJ9DSs2sVwE0iVr++pAHE6QkS9tzcVWozlPifdwMgrU= +github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -649,7 +812,11 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -666,6 +833,7 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= @@ -679,15 +847,18 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE= -github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug= +github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= @@ -696,8 +867,9 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.1 h1:TlEtJq5GvGqMykEwWzbZWjjztF86swFhsPix1i0bkgA= -github.com/prometheus/procfs v0.7.1/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -708,11 +880,15 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 h1:N3Af8f13ooDKcIhsmFT7Z05CStZWu4C7Md0uDEy4q6o= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873/go.mod h1:dmPawKuiAeG/aFYVs2i+Dyosoo7FNcm+Pi8iK6ZUrX8= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -729,15 +905,22 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -746,9 +929,12 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -771,11 +957,14 @@ github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKk github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/testcontainers/testcontainers-go v0.12.0 h1:SK0NryGHIx7aifF6YqReORL18aGAA4bsDPtikDVCEyg= -github.com/testcontainers/testcontainers-go v0.12.0/go.mod h1:SIndOQXZng0IW8iWU1Js0ynrfZ8xcxrTtDfF6rD2pxs= +github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBNGN0TYb/7oKIPVn15JA= +github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -791,14 +980,19 @@ github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7Fw github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -809,6 +1003,7 @@ github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBU github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= @@ -817,6 +1012,10 @@ go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3C go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -826,14 +1025,35 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= @@ -859,8 +1079,11 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -886,7 +1109,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -940,21 +1162,29 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -968,6 +1198,7 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1002,6 +1233,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1028,6 +1260,7 @@ golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1052,10 +1285,12 @@ golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1066,18 +1301,29 @@ golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1094,12 +1340,15 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1115,6 +1364,7 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1139,24 +1389,27 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1217,16 +1470,19 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1237,8 +1493,11 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 h1:myaecH64R0bIEDjNORIel4iXubqzaHU1K2z8ajBwWcM= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1262,9 +1521,14 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1277,8 +1541,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1286,6 +1551,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= @@ -1293,6 +1559,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= @@ -1313,10 +1580,12 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/gotestsum v1.7.0/go.mod h1:V1m4Jw3eBerhI/A6qCxUE07RnCg7ACkKj9BYcAm09V8= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= @@ -1328,22 +1597,57 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= +k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= +k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= +k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= +k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= +k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= +k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= +k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= +k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= +k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= +k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= +k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 2884abafd05e90235de088ae4639bde6d179b6aa Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 18 Apr 2022 13:15:18 +0300 Subject: [PATCH 256/548] [#143] Update go to v1.17 Signed-off-by: Alex Vanin --- .github/workflows/builds.yml | 4 +- .github/workflows/tests.yml | 4 +- Dockerfile | 2 +- go.mod | 90 ++++++++++++++++++++++++++++++------ 4 files changed, 81 insertions(+), 19 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 9532af8..1ac51ce 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -22,7 +22,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Restore Go modules from cache uses: actions/cache@v2 @@ -55,7 +55,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Restore Go modules from cache uses: actions/cache@v2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 42f8b70..f03488d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,7 +35,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Restore Go modules from cache uses: actions/cache@v2 @@ -61,7 +61,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - go_versions: [ '1.16' ] + go_versions: [ '1.17', '1.18' ] fail-fast: false steps: - uses: actions/checkout@v2 diff --git a/Dockerfile b/Dockerfile index 187993e..015e4b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.16-alpine as basebuilder +FROM golang:1.17-alpine as basebuilder RUN apk add --update make bash ca-certificates FROM basebuilder as builder diff --git a/go.mod b/go.mod index 487e2ff..2b4f590 100644 --- a/go.mod +++ b/go.mod @@ -1,34 +1,96 @@ module github.com/nspcc-dev/neofs-http-gw -go 1.16 +go 1.17 require ( - github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/containerd/containerd v1.6.2 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/docker v20.10.14+incompatible // indirect github.com/fasthttp/router v1.4.1 - github.com/gorilla/mux v1.8.0 // indirect - github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect - github.com/magiconair/properties v1.8.6 // indirect - github.com/moby/sys/mount v0.3.2 // indirect github.com/nspcc-dev/neo-go v0.98.0 github.com/nspcc-dev/neofs-api-go/v2 v2.12.1 github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220407103316-e50e6d28280d - github.com/opencontainers/runc v1.1.1 // indirect github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.30.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 - github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/testcontainers/testcontainers-go v0.13.0 github.com/valyala/fasthttp v1.34.0 - go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.18.1 +) + +require ( + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/Microsoft/hcsshim v0.9.2 // indirect + github.com/andybalholm/brotli v1.0.4 // indirect + github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/btcsuite/btcd v0.22.0-beta // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/containerd/cgroups v1.0.3 // indirect + github.com/containerd/containerd v1.6.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/docker v20.10.14+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/klauspost/compress v1.15.0 // indirect + github.com/magiconair/properties v1.8.6 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/moby/sys/mount v0.3.2 // indirect + github.com/moby/sys/mountinfo v0.6.1 // indirect + github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/nspcc-dev/hrw v1.0.9 // indirect + github.com/nspcc-dev/neofs-crypto v0.3.0 // indirect + github.com/nspcc-dev/rfc6979 v0.2.0 // indirect + github.com/onsi/ginkgo v1.16.4 // indirect + github.com/onsi/gomega v1.15.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/opencontainers/runc v1.1.1 // indirect + github.com/pelletier/go-toml v1.9.3 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/urfave/cli v1.22.5 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + go.etcd.io/bbolt v1.3.6 // indirect + go.opencensus.io v0.23.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.7.0 // indirect + golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/text v0.3.7 // indirect google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect + google.golang.org/grpc v1.45.0 // indirect + google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) From 4f2b21a14bbdee8506d129835275413fb844cd4f Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 20 Apr 2022 13:03:06 +0300 Subject: [PATCH 257/548] [#144] Add more verbose error output Signed-off-by: Denis Kirillov --- downloader/download.go | 16 ++++++++-------- uploader/upload.go | 12 ++++++------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 554987f..bf525bd 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -98,7 +98,7 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress *address.Address) { ) if err = tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) - response.Error(r.RequestCtx, "could not fetch and store bearer token", fasthttp.StatusBadRequest) + response.Error(r.RequestCtx, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -162,7 +162,7 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress *address.Address) { }) if err != nil && err != io.EOF { r.log.Error("could not detect Content-Type from payload", zap.Error(err)) - response.Error(r.RequestCtx, "could not detect Content-Type from payload", fasthttp.StatusBadRequest) + response.Error(r.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -316,7 +316,7 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.P res, err := d.search(c, containerID, key, val, object.MatchStringEqual) if err != nil { log.Error("could not search for objects", zap.Error(err)) - response.Error(c, "could not search for objects", fasthttp.StatusBadRequest) + response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -333,7 +333,7 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.P } log.Error("read object list failed", zap.Error(err)) - response.Error(c, "read object list failed", fasthttp.StatusBadRequest) + response.Error(c, "read object list failed: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -372,14 +372,14 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { if err := tokens.StoreBearerToken(c); err != nil { log.Error("could not fetch and store bearer token", zap.Error(err)) - response.Error(c, "could not fetch and store bearer token", fasthttp.StatusBadRequest) + response.Error(c, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } resSearch, err := d.search(c, containerID, attributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error("could not search for objects", zap.Error(err)) - response.Error(c, "could not search for objects", fasthttp.StatusBadRequest) + response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -457,7 +457,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { }) if errIter != nil { log.Error("iterating over selected objects failed", zap.Error(errIter)) - response.Error(c, "iterating over selected objects", fasthttp.StatusBadRequest) + response.Error(c, "iterating over selected objects: "+errIter.Error(), fasthttp.StatusBadRequest) return } else if !called { log.Error("objects not found") @@ -471,7 +471,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { if err != nil { log.Error("file streaming failure", zap.Error(err)) - response.Error(c, "file streaming failure", fasthttp.StatusInternalServerError) + response.Error(c, "file streaming failure: "+err.Error(), fasthttp.StatusInternalServerError) return } } diff --git a/uploader/upload.go b/uploader/upload.go index 7b0798b..966dd21 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -95,12 +95,12 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { epochDuration, err := getEpochDurations(c, u.pool) if err != nil { log.Error("could not get epoch durations from network info", zap.Error(err)) - response.Error(c, "could parse expiration header, try expiration in epoch", fasthttp.StatusBadRequest) + response.Error(c, "could not get epoch durations from network info: "+err.Error(), fasthttp.StatusBadRequest) return } if err = prepareExpirationHeader(filtered, epochDuration); err != nil { - log.Error("could not prepare expiration header", zap.Error(err)) - response.Error(c, "could parse expiration header, try expiration in epoch", fasthttp.StatusBadRequest) + log.Error("could not parse expiration header", zap.Error(err)) + response.Error(c, "could not parse expiration header: "+err.Error(), fasthttp.StatusBadRequest) return } } @@ -144,7 +144,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { if idObj, err = u.pool.PutObject(ctx, prm); err != nil { log.Error("could not store file in neofs", zap.Error(err)) - response.Error(c, "could not store file in neofs", fasthttp.StatusBadRequest) + response.Error(c, "could not store file in neofs: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -153,8 +153,8 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { - log.Error("could not prepare response", zap.Error(err)) - response.Error(c, "could not prepare response", fasthttp.StatusBadRequest) + log.Error("could not encode response", zap.Error(err)) + response.Error(c, "could not encode response", fasthttp.StatusBadRequest) return } From d906732ef4c4de8ba6a7552192b47c06ee0dfbbc Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 21 Apr 2022 18:53:17 +0300 Subject: [PATCH 258/548] [#146] Update default values for HTTP_GW_READ/WRITE_TIMEOUT 10 minute upload and 5 minute download timeouts are long enough to handle average size objects in the real networks. For big data streams these timeouts should be disabled. Signed-off-by: Alex Vanin --- README.md | 5 +++++ config/config.env | 4 ++-- config/config.yaml | 4 ++-- settings.go | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8126473..7a0f80a 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,11 @@ You can tune HTTP read and write buffer sizes as well as timeouts with `HTTP_GW_WEB_WRITE_BUFFER_SIZE` and `HTTP_GW_WEB_WRITE_TIMEOUT` environment variables. +**Note:** to allow upload and download of big data streams, disable read +and write timeouts correspondingly. To do that, set `HTTP_GW_WEB_READ_TIMEOUT=0` +and `HTTP_GW_WEB_WRITE_TIMEOUT=0`. Otherwise, HTTP Gateway will terminate +request with data stream after timeout. + `HTTP_GW_WEB_STREAM_REQUEST_BODY` environment variable can be used to disable request body streaming (effectively it'll make gateway accept file completely first and only then try sending it to NeoFS). diff --git a/config/config.env b/config/config.env index fffa851..564b2dd 100644 --- a/config/config.env +++ b/config/config.env @@ -53,11 +53,11 @@ HTTP_GW_WRITE_BUFFER_SIZE=4096 # the full request including body. The connection's read # deadline is reset when the connection opens, or for # keep-alive connections after the first byte has been read. -HTTP_GW_READ_TIMEOUT=15s +HTTP_GW_READ_TIMEOUT=10m # WriteTimeout is the maximum duration before timing out # writes of the response. It is reset after the request handler # has returned. -HTTP_GW_WRITE_TIMEOUT=1m +HTTP_GW_WRITE_TIMEOUT=5m # StreamRequestBody enables request body streaming, # and calls the handler sooner when given body is # larger then the current limit. diff --git a/config/config.yaml b/config/config.yaml index 26c0d30..7cf3e5f 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -50,12 +50,12 @@ web: # the full request including body. The connection's read # deadline is reset when the connection opens, or for # keep-alive connections after the first byte has been read. - read_timeout: 15s + read_timeout: 10m # WriteTimeout is the maximum duration before timing out # writes of the response. It is reset after the request handler # has returned. - write_timeout: 1m + write_timeout: 5m # StreamRequestBody enables request body streaming, # and calls the handler sooner when given body is diff --git a/settings.go b/settings.go index 624a786..62af367 100644 --- a/settings.go +++ b/settings.go @@ -107,8 +107,8 @@ func settings() *viper.Viper { // web-server: v.SetDefault(cfgWebReadBufferSize, 4096) v.SetDefault(cfgWebWriteBufferSize, 4096) - v.SetDefault(cfgWebReadTimeout, time.Second*15) - v.SetDefault(cfgWebWriteTimeout, time.Minute) + v.SetDefault(cfgWebReadTimeout, time.Minute*10) + v.SetDefault(cfgWebWriteTimeout, time.Minute*5) v.SetDefault(cfgWebStreamRequestBody, true) v.SetDefault(cfgWebMaxRequestBodySize, fasthttp.DefaultMaxRequestBodySize) From 1e3df95eed5737c3325ad1176921b17f18558531 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 21 Apr 2022 18:04:41 +0300 Subject: [PATCH 259/548] [#145] Use application context in NeoFS API requests It is meaningless to use RequestCtx as a context.Context for NeoFS operation, because context won't be closed until application shutdown. Moreover, it also triggers data race detection, because server's done channel, which is accessible for reading from RequestCtx, is set to `nil`. Using application context doesn't change gateway behavior, but it suppresses data race trigger at shutdown. It also allows possibility to set configurable timeouts for NeoFS networking if we will ever need them. Signed-off-by: Alex Vanin --- app.go | 4 ++-- downloader/download.go | 15 +++++++++------ downloader/head.go | 4 ++-- uploader/upload.go | 10 ++++------ 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/app.go b/app.go index 9e669eb..bfdd66b 100644 --- a/app.go +++ b/app.go @@ -208,8 +208,8 @@ func (a *app) Serve(ctx context.Context) { close(a.webDone) }() edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) - uploader := uploader.New(a.log, a.pool, edts) - downloader := downloader.New(a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool) + uploader := uploader.New(ctx, a.log, a.pool, edts) + downloader := downloader.New(ctx, a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool) // Configure router. r := router.New() r.RedirectTrailingSlash = true diff --git a/downloader/download.go b/downloader/download.go index bf525bd..4f405f1 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -31,7 +31,8 @@ import ( type request struct { *fasthttp.RequestCtx - log *zap.Logger + appCtx context.Context + log *zap.Logger } var errObjectNotFound = errors.New("object not found") @@ -106,7 +107,7 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress *address.Address) { prm.SetAddress(*objectAddress) prm.UseBearer(bearerToken(r.RequestCtx)) - rObj, err := clnt.GetObject(r.RequestCtx, prm) + rObj, err := clnt.GetObject(r.appCtx, prm) if err != nil { r.handleNeoFSErr(err, start) return @@ -247,6 +248,7 @@ func (r *request) handleNeoFSErr(err error, start time.Time) { // Downloader is a download request handler. type Downloader struct { + appCtx context.Context log *zap.Logger pool *pool.Pool settings Settings @@ -257,13 +259,14 @@ type Settings struct { } // New creates an instance of Downloader using specified options. -func New(log *zap.Logger, settings Settings, conns *pool.Pool) *Downloader { - return &Downloader{log: log, pool: conns, settings: settings} +func New(ctx context.Context, log *zap.Logger, settings Settings, conns *pool.Pool) *Downloader { + return &Downloader{appCtx: ctx, log: log, pool: conns, settings: settings} } func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { return &request{ RequestCtx: ctx, + appCtx: d.appCtx, log: log, } } @@ -354,7 +357,7 @@ func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string prm.SetFilters(filters) prm.UseBearer(bearerToken(c)) - return d.pool.SearchObjects(c, prm) + return d.pool.SearchObjects(d.appCtx, prm) } // DownloadZipped handles zip by prefix requests. @@ -423,7 +426,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { prm.SetAddress(addr) prm.UseBearer(btoken) - resGet, err = d.pool.GetObject(c, prm) + resGet, err = d.pool.GetObject(d.appCtx, prm) if err != nil { err = fmt.Errorf("get NeoFS object: %v", err) return true diff --git a/downloader/head.go b/downloader/head.go index a299d31..6eb10e6 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -39,7 +39,7 @@ func (r request) headObject(clnt *pool.Pool, objectAddress *address.Address) { prm.SetAddress(*objectAddress) prm.UseBearer(btoken) - obj, err := clnt.HeadObject(r.RequestCtx, prm) + obj, err := clnt.HeadObject(r.appCtx, prm) if err != nil { r.handleNeoFSErr(err, start) return @@ -79,7 +79,7 @@ func (r request) headObject(clnt *pool.Pool, objectAddress *address.Address) { prmRange.SetLength(sz) prmRange.UseBearer(btoken) - return clnt.ObjectRange(r.RequestCtx, prmRange) + return clnt.ObjectRange(r.appCtx, prmRange) }) if err != nil && err != io.EOF { r.handleNeoFSErr(err, start) diff --git a/uploader/upload.go b/uploader/upload.go index 966dd21..e6b71a6 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -31,6 +31,7 @@ const ( // Uploader is an upload request handler. type Uploader struct { + appCtx context.Context log *zap.Logger pool *pool.Pool enableDefaultTimestamp bool @@ -44,8 +45,8 @@ type epochDurations struct { // New creates a new Uploader using specified logger, connection pool and // other options. -func New(log *zap.Logger, conns *pool.Pool, enableDefaultTimestamp bool) *Uploader { - return &Uploader{log, conns, enableDefaultTimestamp} +func New(ctx context.Context, log *zap.Logger, conns *pool.Pool, enableDefaultTimestamp bool) *Uploader { + return &Uploader{ctx, log, conns, enableDefaultTimestamp} } // Upload handles multipart upload request. @@ -134,15 +135,12 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { obj.SetOwnerID(id) obj.SetAttributes(attributes...) - ctx, cancel := context.WithCancel(c) - defer cancel() - var prm pool.PrmObjectPut prm.SetHeader(*obj) prm.SetPayload(file) prm.UseBearer(bt) - if idObj, err = u.pool.PutObject(ctx, prm); err != nil { + if idObj, err = u.pool.PutObject(u.appCtx, prm); err != nil { log.Error("could not store file in neofs", zap.Error(err)) response.Error(c, "could not store file in neofs: "+err.Error(), fasthttp.StatusBadRequest) return From 11283c1c7948d7beca71a6860752f6321c3ad2ea Mon Sep 17 00:00:00 2001 From: Elizaveta Chichindaeva Date: Thu, 21 Apr 2022 11:35:57 +0300 Subject: [PATCH 260/548] [#153] English Check Signed-off-by: Elizaveta Chichindaeva --- CONTRIBUTING.md | 12 +++---- README.md | 74 +++++++++++++++++++++--------------------- downloader/download.go | 10 +++--- misc.go | 2 +- tokens/bearer-token.go | 8 ++--- uploader/filter.go | 8 ++--- 6 files changed, 57 insertions(+), 57 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e73dbf8..5af026c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,14 +9,14 @@ everyone. Please follow the guidelines: - Open an issue first, to discuss a new feature or enhancement. -- Write tests, and make sure the test suite passes locally and on CI. +- Write tests and make sure the test suite passes locally and on CI. -- Open a pull request, and reference the relevant issue(s). +- Open a pull request and reference the relevant issue(s). - Make sure your commits are logically separated and have good comments explaining the details of your change. -- After receiving feedback, amend your commits or add new ones as +- After receiving a feedback, amend your commits or add new ones as appropriate. - **Have fun!** @@ -48,7 +48,7 @@ $ git merge upstream/master ### Create your feature branch Before making code changes, make sure you create a separate branch for these -changes. Maybe you will find it convenient to name branch in +changes. Maybe you will find it convenient to name a branch in `/-` format. ``` @@ -98,7 +98,7 @@ reviewed and approved, it will be merged. ## DCO Sign off All authors to the project retain copyright to their work. However, to ensure -that they are only submitting work that they have rights to, we are requiring +that they are only submitting work that they have rights to, we require everyone to acknowledge this by signing their work. Any copyright notices in this repository should specify the authors as "the @@ -110,7 +110,7 @@ To sign your work, just add a line like this at the end of your commit message: Signed-off-by: Samii Sakisaka ``` -This can easily be done with the `--signoff` option to `git commit`. +This can be easily done with the `--signoff` option to `git commit`. By doing this you state that you can certify the following (from [The Developer Certificate of Origin](https://developercertificate.org/)): diff --git a/README.md b/README.md index 7a0f80a..563fb26 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ # NeoFS HTTP Gateway NeoFS HTTP Gateway bridges NeoFS internal protocol and HTTP standard. -- you can download one file per request from NeoFS Network +- you can download one file per request from the NeoFS Network - you can upload one file per request into the NeoFS Network ## Installation @@ -35,8 +35,8 @@ version Show current version ``` Or you can also use a [Docker -image](https://hub.docker.com/r/nspccdev/neofs-http-gw) provided for released -(and occasionally unreleased) versions of gateway (`:latest` points to the +image](https://hub.docker.com/r/nspccdev/neofs-http-gw) provided for the released +(and occasionally unreleased) versions of the gateway (`:latest` points to the latest stable release). ## Execution @@ -47,8 +47,8 @@ can be done either via `-p` parameter or via `HTTP_GW_PEERS__ADDRESS` and `HTTP_GW_PEERS__WEIGHT` environment variables (the gate supports multiple NeoFS nodes with weighted load balancing). -If you're launching HTTP gateway in bundle with [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env), -you can get an IP address of the node in output of `make hosts` command +If you launch HTTP gateway in bundle with [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env), +you can get the IP address of the node in the output of `make hosts` command (with s0*.neofs.devenv name). These two commands are functionally equivalent, they run the gate with one @@ -89,9 +89,9 @@ This command will make gateway use 192.168.130.71 while it is healthy. Otherwise 192.168.130.72 for 90% of requests and 192.168.130.73 for remaining 10%. ### Keys -You can provide wallet via `--wallet` or `-w` flag also you can specify account address using `--address` -(if no address provided default one will be used). If wallet is used you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt wallet. -If no wallet provided gateway autogenerates key pair it will use for NeoFS requests. +You can provide a wallet via `--wallet` or `-w` flag. You can also specify the account address using `--address` +(if no address provided default one will be used). If wallet is used, you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt the wallet. +If no wallet provided, the gateway autogenerates a key pair it will use for NeoFS requests. ``` $ neofs-http-gw -p $NEOFS_NODE -w $WALLET_PATH --address $ACCOUNT_ADDRESS ``` @@ -105,10 +105,10 @@ $ neofs-http-gw -p 192.168.130.72:8080 -w wallet.json --address NfgHwwTi3wHAS8aF Gateway binds to `0.0.0.0:8082` by default and you can change that with `--listen_address` option. -It can also provide TLS interface for its users, just specify paths to key and +It can also provide TLS interface for its users, just specify paths to the key and certificate files via `--tls_key` and `--tls_certificate` parameters. Note -that using these options makes gateway TLS-only, if you need to serve both TLS -and plain text HTTP you either have to run two gateway instances or use some +that using these options makes gateway TLS-only. If you need to serve both TLS +and plain text HTTP, you either have to run two gateway instances or use some external redirecting solution. Example to bind to `192.168.130.130:443` and serve TLS there: @@ -131,7 +131,7 @@ and `HTTP_GW_WEB_WRITE_TIMEOUT=0`. Otherwise, HTTP Gateway will terminate request with data stream after timeout. `HTTP_GW_WEB_STREAM_REQUEST_BODY` environment variable can be used to disable -request body streaming (effectively it'll make gateway accept file completely +request body streaming (effectively it'll make the gateway accept the file completely first and only then try sending it to NeoFS). `HTTP_GW_WEB_MAX_REQUEST_BODY_SIZE` controls maximum request body size @@ -145,17 +145,17 @@ variable to control this behavior. ### Monitoring and metrics -Pprof and Prometheus are integrated into the gateway, but not enabled by +Pprof and Prometheus are integrated into the gateway, but they are not enabled by default. To enable them use `--pprof` and `--metrics` flags or `HTTP_GW_PPROF`/`HTTP_GW_METRICS` environment variables. ### Timeouts You can tune gRPC interface parameters with `--connect_timeout` (for -connection to node) and `--request_timeout` (for request processing over +connection to a node) and `--request_timeout` (for request processing over established connection) options. -gRPC-level checks allow gateway to detect dead peers, but it declares them +gRPC-level checks allow the gateway to detect dead peers, but it declares them unhealthy at pool level once per `--rebalance_timer` interval, so check for it if needed. @@ -194,16 +194,16 @@ supported. Before uploading or downloading a file make sure you have a prepared container. You can create it with instructions below. -Also in case of downloading you need to have a file inside a container. +Also ,in case of downloading, you need to have a file inside a container. #### Create a container -You can create a container via [neofs-cli](https://github.com/nspcc-dev/neofs-node/releases): +You can create a container via [neofs-cli](https://github.com/nspcc-dev/neofs-node/releases): ``` $ neofs-cli -r $NEOFS_NODE -k $KEY container create --policy $POLICY --basic-acl $ACL ``` -where `$KEY` can be a path to private key file (as raw bytes), a hex string or -(unencrypted) WIF string, +where `$KEY` can be a path to a private key file (as raw bytes), a hex string or +a (unencrypted) WIF string, `$ACL` -- hex encoded basic ACL value or keywords 'private, 'public-read', 'public-read-write' and `$POLICY` -- QL-encoded or JSON-encoded placement policy or path to file with it @@ -212,7 +212,7 @@ For example: $ neofs-cli -r 192.168.130.72:8080 -k 6PYLKJhiSub5imt6WCVy6Quxtd9xu176omev1vWYovzkAQCTSQabAAQXii container create --policy "REP 3" --basic-acl public --await ``` -If you launched nodes via [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env) +If you have launched nodes via [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env), you can get the key value from `wallets/wallet.json` or write the path to the file `wallets/wallet.key`. @@ -279,7 +279,7 @@ $ wget http://localhost:8082/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6Bf $ wget http://localhost:8082/get_by_attribute/88GdaZFTcYJn1dqiSECss8kKPmmun6d6BfvC4zhwfLYM/FileName/cat%25jpeg # means 'cat%jpeg' ``` -Some other user-defined attribute: +Some other user-defined attributes: ``` $ wget http://localhost:8082/get_by_attribute/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/Ololo/100500 @@ -332,10 +332,10 @@ set of reply headers generated using the following rules: ### Uploading -You can POST files to `/upload/$CID` path where `$CID` is container ID. The +You can POST files to `/upload/$CID` path where `$CID` is a container ID. The request must contain multipart form with mandatory `filename` parameter. Only one part in multipart form will be processed, so to upload another file just -issue new POST request. +issue a new POST request. Example request: @@ -345,7 +345,7 @@ $ curl -F 'file=@cat.jpeg;filename=cat.jpeg' http://localhost:8082/upload/Dxhf4P Chunked encoding is supported by the server (but check for request read timeouts if you're planning some streaming). You can try streaming support -with large file piped through named FIFO pipe: +with a large file piped through named FIFO pipe: ``` $ mkfifo pipe @@ -383,7 +383,7 @@ which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide exp --- -For successful uploads you get JSON data in reply body with container and +For successful uploads you get JSON data in reply body with a container and object ID, like this: ``` { @@ -396,7 +396,7 @@ object ID, like this: You can always upload files to public containers (open for anyone to put objects into), but for restricted containers you need to explicitly allow PUT -operations for request signed with your HTTP Gateway keys. +operations for a request signed with your HTTP Gateway keys. If your don't want to manage gateway's secret keys and adjust eACL rules when gateway configuration changes (new gate, key rotation, etc) or you plan to use @@ -410,23 +410,23 @@ documentation for more details). There are two options to pass them to gateway: credentials field * "Bearer" cookie with base64-encoded token contents -For example you have a mobile application frontend with a backend part storing -data in NeoFS. When user authorizes in mobile app, the backend issues a NeoFS -Bearer token and provides it to the frontend. Then the mobile app may generate +For example, you have a mobile application frontend with a backend part storing +data in NeoFS. When a user authorizes in the mobile app, the backend issues a NeoFS +Bearer token and provides it to the frontend. Then, the mobile app may generate some data and upload it via any available NeoFS HTTP Gateway by adding the corresponding header to the upload request. Accessing the ACL protected data works the same way. ##### Example -In order to generate bearer token, you need to know container owner key and -address of sender who will be do request to NeoFS (in our case it's gateway wallet address). +In order to generate a bearer token, you need to know the container owner key and +the address of the sender who will do the request to NeoFS (in our case, it's a gateway wallet address). Suppose we have: * **KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr** (container owner key) * **NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3** (token owner address) * **BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K** (container id) -Firstly we need to encode container id and sender address to base64 (now it's base58). +Firstly, we need to encode the container id and the sender address to base64 (now it's base58). So use **base58** and **base64** utils. 1. Encoding container id: @@ -441,7 +441,7 @@ $ echo 'NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3' | base58 --decode | base64 # output: NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg== ``` -Now we can form Bearer token (10000 is liftetime expiration in epoch) and save it to **bearer.json**: +Now, we can form a Bearer token (10000 is liftetime expiration in epoch) and save it to **bearer.json**: ``` { "body": { @@ -468,17 +468,17 @@ Now we can form Bearer token (10000 is liftetime expiration in epoch) and save i } ``` -Then sign it with container owner key: +Next, sign it with the container owner key: ``` $ neofs-cli util sign bearer-token --from bearer.json --to signed.json -k KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr ``` -Encoding to base64 to use via header: +Encoding to base64 to use via the header: ``` $ base64 -w 0 signed.json # output: Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw== ``` -After that Bearer token can be used: +After that, the Bearer token can be used: ``` $ curl -F 'file=@cat.jpeg;filename=cat.jpeg' -H "Authorization: Bearer Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw==" \ @@ -500,7 +500,7 @@ For example: $ neofs-cli --key KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr --basic-acl 0x0FFFCFFF -r 192.168.130.72:8080 container create --policy "REP 3" --await ``` -To deny access to the container without a token, set the eACL rules: +To deny access to a container without a token, set the eACL rules: ``` $ neofs-cli --key KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr -r 192.168.130.72:8080 container set-eacl --table eacl.json --await --cid BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K ``` diff --git a/downloader/download.go b/downloader/download.go index 4f405f1..2afa343 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -66,8 +66,8 @@ type readCloser struct { io.Closer } -// initializes io.Reader with limited size and detects Content-Type from it. -// Returns r's error directly. Also returns processed data. +// initializes io.Reader with the limited size and detects Content-Type from it. +// Returns r's error directly. Also returns the processed data. func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (string, []byte, error) { if maxSize > sizeToDetectType { maxSize = sizeToDetectType @@ -167,7 +167,7 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress *address.Address) { return } - // reset payload reader since part of the data has been read + // reset payload reader since a part of the data has been read var headReader io.Reader = bytes.NewReader(payloadHead) if err != io.EOF { // otherwise, we've already read full payload @@ -276,7 +276,7 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { d.byAddress(c, request.receiveFile) } -// byAddress is wrapper for function (e.g. request.headObject, request.receiveFile) that +// byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) { var ( @@ -300,7 +300,7 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { d.byAttribute(c, request.receiveFile) } -// byAttribute is wrapper similar to byAddress. +// byAttribute is a wrapper similar to byAddress. func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) { var ( httpStatus = fasthttp.StatusBadRequest diff --git a/misc.go b/misc.go index e82945a..1edee97 100644 --- a/misc.go +++ b/misc.go @@ -5,6 +5,6 @@ package main const Prefix = "HTTP_GW" var ( - // Version is gateway version. + // Version is the gateway version. Version = "dev" ) diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index 48b109e..8899b90 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -26,7 +26,7 @@ const ( // return // } -// BearerTokenFromHeader extracts bearer token from Authorization request header. +// BearerTokenFromHeader extracts a bearer token from Authorization request header. func BearerTokenFromHeader(h *fasthttp.RequestHeader) []byte { auth := h.Peek(fasthttp.HeaderAuthorization) if auth == nil || !bytes.HasPrefix(auth, []byte(bearerTokenHdr)) { @@ -38,7 +38,7 @@ func BearerTokenFromHeader(h *fasthttp.RequestHeader) []byte { return auth } -// BearerTokenFromCookie extracts bearer token from cookies. +// BearerTokenFromCookie extracts a bearer token from cookies. func BearerTokenFromCookie(h *fasthttp.RequestHeader) []byte { auth := h.Cookie(bearerTokenHdr) if len(auth) == 0 { @@ -48,7 +48,7 @@ func BearerTokenFromCookie(h *fasthttp.RequestHeader) []byte { return auth } -// StoreBearerToken extracts bearer token from header or cookie and stores +// StoreBearerToken extracts a bearer token from the header or cookie and stores // it in the request context. func StoreBearerToken(ctx *fasthttp.RequestCtx) error { tkn, err := fetchBearerToken(ctx) @@ -60,7 +60,7 @@ func StoreBearerToken(ctx *fasthttp.RequestCtx) error { return nil } -// LoadBearerToken returns bearer token stored in context given (if it's +// LoadBearerToken returns a bearer token stored in the context given (if it's // present there). func LoadBearerToken(ctx context.Context) (*token.BearerToken, error) { if tkn, ok := ctx.Value(bearerTokenKey).(*token.BearerToken); ok && tkn != nil { diff --git a/uploader/filter.go b/uploader/filter.go index 98046c4..0152920 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -15,7 +15,7 @@ import ( var neofsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")} func systemTranslator(key, prefix []byte) []byte { - // replace specified prefix with `__NEOFS__` + // replace the specified prefix with `__NEOFS__` key = bytes.Replace(key, prefix, []byte(utils.SystemAttributePrefix), 1) // replace `-` with `_` @@ -30,12 +30,12 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]str prefix := []byte(utils.UserAttributeHeaderPrefix) header.VisitAll(func(key, val []byte) { - // checks that key and val not empty + // checks that the key and the val not empty if len(key) == 0 || len(val) == 0 { return } - // checks that key has attribute prefix + // checks that the key has attribute prefix if !bytes.HasPrefix(key, prefix) { return } @@ -51,7 +51,7 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]str } } - // checks that attribute key not empty + // checks that the attribute key is not empty if len(key) == 0 { return } From 2b780c1772cc8b331e702d2c834183fec9dc4757 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 19 Apr 2022 18:46:51 +0300 Subject: [PATCH 261/548] [#142] Update SDK Signed-off-by: Denis Kirillov --- downloader/download.go | 32 +++++++++++++++++++------------- downloader/head.go | 14 ++++++++++---- go.mod | 3 ++- go.sum | 7 +++++-- integration_test.go | 12 ++++++------ tokens/bearer-token.go | 10 +++++----- tokens/bearer-token_test.go | 29 +++++++++++------------------ uploader/upload.go | 30 ++++++++++++++++++------------ 8 files changed, 76 insertions(+), 61 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 2afa343..542e85d 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -19,12 +19,12 @@ import ( "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" + "github.com/nspcc-dev/neofs-sdk-go/bearer" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object/address" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/pool" - "github.com/nspcc-dev/neofs-sdk-go/token" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -105,7 +105,9 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress *address.Address) { var prm pool.PrmObjectGet prm.SetAddress(*objectAddress) - prm.UseBearer(bearerToken(r.RequestCtx)) + if btoken := bearerToken(r.RequestCtx); btoken != nil { + prm.UseBearer(*btoken) + } rObj, err := clnt.GetObject(r.appCtx, prm) if err != nil { @@ -215,7 +217,7 @@ func title(str string) string { return string(r0) + str[size:] } -func bearerToken(ctx context.Context) *token.BearerToken { +func bearerToken(ctx context.Context) *bearer.Token { if tkn, err := tokens.LoadBearerToken(ctx); err == nil { return tkn } @@ -309,8 +311,8 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.P val, _ = url.QueryUnescape(c.UserValue("attr_val").(string)) log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ) - containerID := cid.New() - if err := containerID.Parse(scid); err != nil { + containerID := new(cid.ID) + if err := containerID.DecodeString(scid); err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", httpStatus) return @@ -341,8 +343,8 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.P } var addrObj address.Address - addrObj.SetContainerID(containerID) - addrObj.SetObjectID(&buf[0]) + addrObj.SetContainerID(*containerID) + addrObj.SetObjectID(buf[0]) f(*d.newRequest(c, log), d.pool, &addrObj) } @@ -355,7 +357,9 @@ func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string var prm pool.PrmObjectSearch prm.SetContainerID(*cid) prm.SetFilters(filters) - prm.UseBearer(bearerToken(c)) + if btoken := bearerToken(c); btoken != nil { + prm.UseBearer(*btoken) + } return d.pool.SearchObjects(d.appCtx, prm) } @@ -366,8 +370,8 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string)) log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) - containerID := cid.New() - if err := containerID.Parse(scid); err != nil { + containerID := new(cid.ID) + if err := containerID.DecodeString(scid); err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return @@ -405,7 +409,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { bufZip []byte ) - addr.SetContainerID(containerID) + addr.SetContainerID(*containerID) btoken := bearerToken(c) empty := true @@ -420,11 +424,13 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { empty = false - addr.SetObjectID(&id) + addr.SetObjectID(id) var prm pool.PrmObjectGet prm.SetAddress(addr) - prm.UseBearer(btoken) + if btoken != nil { + prm.UseBearer(*btoken) + } resGet, err = d.pool.GetObject(d.appCtx, prm) if err != nil { diff --git a/downloader/head.go b/downloader/head.go index 6eb10e6..d33d6d8 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -37,7 +37,9 @@ func (r request) headObject(clnt *pool.Pool, objectAddress *address.Address) { var prm pool.PrmObjectHead prm.SetAddress(*objectAddress) - prm.UseBearer(btoken) + if btoken != nil { + prm.UseBearer(*btoken) + } obj, err := clnt.HeadObject(r.appCtx, prm) if err != nil { @@ -77,7 +79,9 @@ func (r request) headObject(clnt *pool.Pool, objectAddress *address.Address) { var prmRange pool.PrmObjectRange prmRange.SetAddress(*objectAddress) prmRange.SetLength(sz) - prmRange.UseBearer(btoken) + if btoken != nil { + prmRange.UseBearer(*btoken) + } return clnt.ObjectRange(r.appCtx, prmRange) }) @@ -90,9 +94,11 @@ func (r request) headObject(clnt *pool.Pool, objectAddress *address.Address) { } func idsToResponse(resp *fasthttp.Response, obj *object.Object) { - resp.Header.Set(hdrObjectID, obj.ID().String()) + objID, _ := obj.ID() + cnrID, _ := obj.ContainerID() + resp.Header.Set(hdrObjectID, objID.String()) resp.Header.Set(hdrOwnerID, obj.OwnerID().String()) - resp.Header.Set(hdrContainerID, obj.ContainerID().String()) + resp.Header.Set(hdrContainerID, cnrID.String()) } // HeadByAddress handles head requests using simple cid/oid format. diff --git a/go.mod b/go.mod index 2b4f590..f2b86b7 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.98.0 github.com/nspcc-dev/neofs-api-go/v2 v2.12.1 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220407103316-e50e6d28280d + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220421125737-6e81e13e1bff github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.30.0 github.com/spf13/pflag v1.0.5 @@ -56,6 +56,7 @@ require ( github.com/nspcc-dev/hrw v1.0.9 // indirect github.com/nspcc-dev/neofs-crypto v0.3.0 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect + github.com/nspcc-dev/tzhash v1.5.2 // indirect github.com/onsi/ginkgo v1.16.4 // indirect github.com/onsi/gomega v1.15.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect diff --git a/go.sum b/go.sum index 818ae84..c869b51 100644 --- a/go.sum +++ b/go.sum @@ -745,11 +745,13 @@ github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BE github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220407103316-e50e6d28280d h1:OHyq8+zyQtARFWj3quRPabcfQWJZEiU7HYp6QGCSjaM= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220407103316-e50e6d28280d/go.mod h1:Hl7a1l0ntZ4b1ZABpGX6fuAuFS3c6+hyMCUNVvZv/w4= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220421125737-6e81e13e1bff h1:rnkApn6vXUDcN9e/AsS/gaQZSe24VulI5IYrOkkvutM= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220421125737-6e81e13e1bff/go.mod h1:cQKdlr9Gmp5jxbOJ78S714i1AycfYUzpVddxVUD48WM= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= +github.com/nspcc-dev/tzhash v1.5.2 h1:GuIQPOY2xpl5ZE1pbUbz+QdKXVOTyzbbxSVv0nBfa98= +github.com/nspcc-dev/tzhash v1.5.2/go.mod h1:gwAx6mcsbkfY+JVp+PovoP2Gvw6y57W8dj7zDHKOhzI= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -1316,6 +1318,7 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/integration_test.go b/integration_test.go index 174db56..d8f7133 100644 --- a/integration_test.go +++ b/integration_test.go @@ -108,16 +108,16 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci err = json.NewDecoder(resp.Body).Decode(addr) require.NoError(t, err) - err = CID.Parse(addr.CID) + err = CID.DecodeString(addr.CID) require.NoError(t, err) - id := oid.NewID() - err = id.Parse(addr.OID) + id := new(oid.ID) + err = id.DecodeString(addr.OID) require.NoError(t, err) objectAddress := address.NewAddress() - objectAddress.SetContainerID(CID) - objectAddress.SetObjectID(id) + objectAddress.SetContainerID(*CID) + objectAddress.SetObjectID(*id) payload := bytes.NewBuffer(nil) @@ -320,7 +320,7 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool) ( func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, content string, attributes map[string]string) *oid.ID { obj := object.New() - obj.SetContainerID(CID) + obj.SetContainerID(*CID) obj.SetOwnerID(clientPool.OwnerID()) var attrs []object.Attribute diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index 8899b90..672138d 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -7,7 +7,7 @@ import ( "errors" "fmt" - "github.com/nspcc-dev/neofs-sdk-go/token" + "github.com/nspcc-dev/neofs-sdk-go/bearer" "github.com/valyala/fasthttp" ) @@ -62,14 +62,14 @@ func StoreBearerToken(ctx *fasthttp.RequestCtx) error { // LoadBearerToken returns a bearer token stored in the context given (if it's // present there). -func LoadBearerToken(ctx context.Context) (*token.BearerToken, error) { - if tkn, ok := ctx.Value(bearerTokenKey).(*token.BearerToken); ok && tkn != nil { +func LoadBearerToken(ctx context.Context) (*bearer.Token, error) { + if tkn, ok := ctx.Value(bearerTokenKey).(*bearer.Token); ok && tkn != nil { return tkn, nil } return nil, errors.New("found empty bearer token") } -func fetchBearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { +func fetchBearerToken(ctx *fasthttp.RequestCtx) (*bearer.Token, error) { // ignore empty value if ctx == nil { return nil, nil @@ -78,7 +78,7 @@ func fetchBearerToken(ctx *fasthttp.RequestCtx) (*token.BearerToken, error) { lastErr error buf []byte - tkn = new(token.BearerToken) + tkn = new(bearer.Token) ) for _, parse := range []fromHandler{BearerTokenFromHeader, BearerTokenFromCookie} { if buf = parse(&ctx.Request.Header); buf == nil { diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index 54258c2..5ecf1dd 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -4,8 +4,8 @@ import ( "encoding/base64" "testing" - "github.com/nspcc-dev/neofs-sdk-go/owner" - "github.com/nspcc-dev/neofs-sdk-go/token" + "github.com/nspcc-dev/neofs-sdk-go/bearer" + "github.com/nspcc-dev/neofs-sdk-go/user" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" ) @@ -59,16 +59,12 @@ func Test_fromHeader(t *testing.T) { } func Test_fetchBearerToken(t *testing.T) { - uid := owner.NewID() + var uid user.ID - tkn := new(token.BearerToken) - tkn.SetOwner(uid) + tkn := new(bearer.Token) + tkn.SetOwnerID(uid) - data, err := tkn.Marshal() - - require.NoError(t, err) - - t64 := base64.StdEncoding.EncodeToString(data) + t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) require.NotEmpty(t, t64) cases := []struct { @@ -78,7 +74,7 @@ func Test_fetchBearerToken(t *testing.T) { header string error string - expect *token.BearerToken + expect *bearer.Token }{ {name: "empty"}, @@ -137,15 +133,12 @@ func makeTestRequest(cookie, header string) *fasthttp.RequestCtx { } func Test_checkAndPropagateBearerToken(t *testing.T) { - uid := owner.NewID() + var uid user.ID - tkn := new(token.BearerToken) - tkn.SetOwner(uid) + tkn := new(bearer.Token) + tkn.SetOwnerID(uid) - data, err := tkn.Marshal() - require.NoError(t, err) - - t64 := base64.StdEncoding.EncodeToString(data) + t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) require.NotEmpty(t, t64) ctx := makeTestRequest(t64, "") diff --git a/uploader/upload.go b/uploader/upload.go index e6b71a6..3910eac 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -12,14 +12,14 @@ import ( "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" + "github.com/nspcc-dev/neofs-sdk-go/bearer" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object/address" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/nspcc-dev/neofs-sdk-go/owner" "github.com/nspcc-dev/neofs-sdk-go/pool" - "github.com/nspcc-dev/neofs-sdk-go/token" + "github.com/nspcc-dev/neofs-sdk-go/user" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -56,7 +56,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { file MultipartFile idObj *oid.ID addr = address.NewAddress() - idCnr = cid.New() + idCnr = new(cid.ID) scid, _ = c.UserValue("cid").(string) log = u.log.With(zap.String("cid", scid)) bodyStream = c.RequestBodyStream() @@ -67,7 +67,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest) return } - if err = idCnr.Parse(scid); err != nil { + if err = idCnr.DecodeString(scid); err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return @@ -131,14 +131,17 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { id, bt := u.fetchOwnerAndBearerToken(c) obj := object.New() - obj.SetContainerID(idCnr) + obj.SetContainerID(*idCnr) obj.SetOwnerID(id) obj.SetAttributes(attributes...) var prm pool.PrmObjectPut prm.SetHeader(*obj) prm.SetPayload(file) - prm.UseBearer(bt) + + if bt != nil { + prm.UseBearer(*bt) + } if idObj, err = u.pool.PutObject(u.appCtx, prm); err != nil { log.Error("could not store file in neofs", zap.Error(err)) @@ -146,8 +149,8 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { return } - addr.SetObjectID(idObj) - addr.SetContainerID(idCnr) + addr.SetObjectID(*idObj) + addr.SetContainerID(*idCnr) // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { @@ -173,9 +176,10 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { c.Response.Header.SetContentType(jsonHeader) } -func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*owner.ID, *token.BearerToken) { +func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*user.ID, *bearer.Token) { if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil { - return tkn.Issuer(), tkn + issuer, _ := tkn.Issuer() + return &issuer, tkn } return u.pool.OwnerID(), nil } @@ -186,9 +190,11 @@ type putResponse struct { } func newPutResponse(addr *address.Address) *putResponse { + objID, _ := addr.ObjectID() + cnrID, _ := addr.ContainerID() return &putResponse{ - ObjectID: addr.ObjectID().String(), - ContainerID: addr.ContainerID().String(), + ObjectID: objID.String(), + ContainerID: cnrID.String(), } } From a42606742a5360523a1098abb04ed3ff15f1fdb3 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 20 Apr 2022 12:17:20 +0300 Subject: [PATCH 262/548] [#142] Support resolving container nicename Signed-off-by: Denis Kirillov --- README.md | 2 + app.go | 58 ++++++++++++++--- config/config.env | 5 ++ config/config.yaml | 6 ++ downloader/download.go | 63 ++++++++++++------- go.mod | 4 ++ go.sum | 3 + integration_test.go | 135 ++++++++++++++++++++++++++++----------- resolver/neofs.go | 46 ++++++++++++++ resolver/resolver.go | 140 +++++++++++++++++++++++++++++++++++++++++ settings.go | 13 ++++ uploader/upload.go | 23 ++++--- utils/params.go | 13 ++++ utils/util.go | 21 +++++++ 14 files changed, 458 insertions(+), 74 deletions(-) create mode 100644 resolver/neofs.go create mode 100644 resolver/resolver.go create mode 100644 utils/params.go create mode 100644 utils/util.go diff --git a/README.md b/README.md index 563fb26..94aa369 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,8 @@ and upload objects with it, but deleting, searching, managing ACLs, creating containers and other activities are not supported and not planned to be supported. +**Note:** in all download/upload routes you can use container name instead of it's id (`$CID`), but resolvers must be configured properly (see [configs](./config) for examples). + ### Preparation Before uploading or downloading a file make sure you have a prepared container. diff --git a/app.go b/app.go index bfdd66b..26201c6 100644 --- a/app.go +++ b/app.go @@ -13,8 +13,10 @@ import ( "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/nspcc-dev/neofs-http-gw/downloader" + "github.com/nspcc-dev/neofs-http-gw/resolver" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/uploader" + "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/spf13/viper" "github.com/valyala/fasthttp" @@ -28,6 +30,7 @@ type ( cfg *viper.Viper webServer *fasthttp.Server webDone chan struct{} + resolver *resolver.ContainerResolver } // App is an interface for the main gateway function. @@ -127,9 +130,39 @@ func newApp(ctx context.Context, opt ...Option) App { if err != nil { a.log.Fatal("failed to dial pool", zap.Error(err)) } + + resolveCfg := &resolver.Config{ + NeoFS: resolver.NewNeoFSResolver(a.pool), + RPCAddress: a.cfg.GetString(cfgRPCEndpoint), + } + + order := a.cfg.GetStringSlice(cfgResolveOrder) + if resolveCfg.RPCAddress == "" { + order = remove(order, resolver.NNSResolver) + a.log.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint)) + } + + if len(order) != 0 { + a.resolver, err = resolver.NewResolver(order, resolveCfg) + if err != nil { + a.log.Fatal("failed to create resolver", zap.Error(err)) + } + } else { + a.log.Info("container resolver is disabled") + } + return a } +func remove(list []string, element string) []string { + for i, item := range list { + if item == element { + return append(list[:i], list[i+1:]...) + } + } + return list +} + func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) { walletPath := a.cfg.GetString(cmdWallet) if len(walletPath) == 0 { @@ -208,8 +241,9 @@ func (a *app) Serve(ctx context.Context) { close(a.webDone) }() edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) - uploader := uploader.New(ctx, a.log, a.pool, edts) - downloader := downloader.New(ctx, a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool) + uploadRoutes := uploader.New(ctx, a.AppParams(), edts) + downloadSettings := downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)} + downloadRoutes := downloader.New(ctx, a.AppParams(), downloadSettings) // Configure router. r := router.New() r.RedirectTrailingSlash = true @@ -219,15 +253,15 @@ func (a *app) Serve(ctx context.Context) { r.MethodNotAllowed = func(r *fasthttp.RequestCtx) { response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.logger(uploader.Upload)) + r.POST("/upload/{cid}", a.logger(uploadRoutes.Upload)) a.log.Info("added path /upload/{cid}") - r.GET("/get/{cid}/{oid}", a.logger(downloader.DownloadByAddress)) - r.HEAD("/get/{cid}/{oid}", a.logger(downloader.HeadByAddress)) + r.GET("/get/{cid}/{oid}", a.logger(downloadRoutes.DownloadByAddress)) + r.HEAD("/get/{cid}/{oid}", a.logger(downloadRoutes.HeadByAddress)) a.log.Info("added path /get/{cid}/{oid}") - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.DownloadByAttribute)) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.HeadByAttribute)) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.DownloadByAttribute)) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.HeadByAttribute)) a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") - r.GET("/zip/{cid}/{prefix:*}", a.logger(downloader.DownloadZipped)) + r.GET("/zip/{cid}/{prefix:*}", a.logger(downloadRoutes.DownloadZipped)) a.log.Info("added path /zip/{cid}/{prefix}") // enable metrics if a.cfg.GetBool(cmdMetrics) { @@ -267,3 +301,11 @@ func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { h(ctx) }) } + +func (a *app) AppParams() *utils.AppParams { + return &utils.AppParams{ + Logger: a.log, + Pool: a.pool, + Resolver: a.resolver, + } +} diff --git a/config/config.env b/config/config.env index 564b2dd..a6bd656 100644 --- a/config/config.env +++ b/config/config.env @@ -66,6 +66,11 @@ HTTP_GW_STREAM_REQUEST_BODY=true # The server rejects requests with bodies exceeding this limit. HTTP_GW_MAX_REQUEST_BODY_SIZE=4194304 +# RPC endpoint to be able to use nns container resolving. +HTTP_GW_RPC_ENDPOINT=http://morph-chain.neofs.devenv:30333 +# The order in which resolvers are used to find an container id by name. +HTTP_GW_RESOLVE_ORDER="nns dns" + # Create timestamp for object if it isn't provided by header. HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=false diff --git a/config/config.yaml b/config/config.yaml index 7cf3e5f..9cdb7ce 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -66,6 +66,12 @@ web: # The server rejects requests with bodies exceeding this limit. max_request_body_size: 4194304 +# RPC endpoint to be able to use nns container resolving. +rpc_endpoint: http://morph-chain.neofs.devenv:30333 +# The order in which resolvers are used to find an container id by name. +resolve_order: + - nns + - dns upload_header: use_default_timestamp: false # Create timestamp for object if it isn't provided by header. diff --git a/downloader/download.go b/downloader/download.go index 542e85d..5456f91 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -16,6 +16,7 @@ import ( "unicode" "unicode/utf8" + "github.com/nspcc-dev/neofs-http-gw/resolver" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" @@ -250,10 +251,11 @@ func (r *request) handleNeoFSErr(err error, start time.Time) { // Downloader is a download request handler. type Downloader struct { - appCtx context.Context - log *zap.Logger - pool *pool.Pool - settings Settings + appCtx context.Context + log *zap.Logger + pool *pool.Pool + containerResolver *resolver.ContainerResolver + settings Settings } type Settings struct { @@ -261,8 +263,14 @@ type Settings struct { } // New creates an instance of Downloader using specified options. -func New(ctx context.Context, log *zap.Logger, settings Settings, conns *pool.Pool) *Downloader { - return &Downloader{appCtx: ctx, log: log, pool: conns, settings: settings} +func New(ctx context.Context, params *utils.AppParams, settings Settings) *Downloader { + return &Downloader{ + appCtx: ctx, + log: params.Logger, + pool: params.Pool, + settings: settings, + containerResolver: params.Resolver, + } } func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { @@ -282,18 +290,29 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { // prepares request and object address to it. func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) { var ( - addr = address.NewAddress() idCnr, _ = c.UserValue("cid").(string) idObj, _ = c.UserValue("oid").(string) - val = strings.Join([]string{idCnr, idObj}, "/") log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj)) ) - if err := addr.Parse(val); err != nil { - log.Error("wrong object address", zap.Error(err)) - response.Error(c, "wrong object address", fasthttp.StatusBadRequest) + + cnrID, err := utils.GetContainerID(d.appCtx, idCnr, d.containerResolver) + if err != nil { + log.Error("wrong container id", zap.Error(err)) + response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } + objID := new(oid.ID) + if err = objID.DecodeString(idObj); err != nil { + log.Error("wrong object id", zap.Error(err)) + response.Error(c, "wrong object id", fasthttp.StatusBadRequest) + return + } + + addr := address.NewAddress() + addr.SetContainerID(*cnrID) + addr.SetObjectID(*objID) + f(*d.newRequest(c, log), d.pool, addr) } @@ -305,16 +324,16 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { // byAttribute is a wrapper similar to byAddress. func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) { var ( - httpStatus = fasthttp.StatusBadRequest - scid, _ = c.UserValue("cid").(string) - key, _ = url.QueryUnescape(c.UserValue("attr_key").(string)) - val, _ = url.QueryUnescape(c.UserValue("attr_val").(string)) - log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) + scid, _ = c.UserValue("cid").(string) + key, _ = url.QueryUnescape(c.UserValue("attr_key").(string)) + val, _ = url.QueryUnescape(c.UserValue("attr_val").(string)) + log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ) - containerID := new(cid.ID) - if err := containerID.DecodeString(scid); err != nil { + + containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver) + if err != nil { log.Error("wrong container id", zap.Error(err)) - response.Error(c, "wrong container id", httpStatus) + response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } @@ -370,14 +389,14 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string)) log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) - containerID := new(cid.ID) - if err := containerID.DecodeString(scid); err != nil { + containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver) + if err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } - if err := tokens.StoreBearerToken(c); err != nil { + if err = tokens.StoreBearerToken(c); err != nil { log.Error("could not fetch and store bearer token", zap.Error(err)) response.Error(c, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return diff --git a/go.mod b/go.mod index f2b86b7..bf5bf37 100644 --- a/go.mod +++ b/go.mod @@ -42,8 +42,10 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.8.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/holiman/uint256 v1.2.0 // indirect github.com/klauspost/compress v1.15.0 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect @@ -53,6 +55,7 @@ require ( github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect + github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02 // indirect github.com/nspcc-dev/hrw v1.0.9 // indirect github.com/nspcc-dev/neofs-crypto v0.3.0 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect @@ -84,6 +87,7 @@ require ( go.uber.org/multierr v1.7.0 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect diff --git a/go.sum b/go.sum index c869b51..0ee3fb9 100644 --- a/go.sum +++ b/go.sum @@ -549,6 +549,7 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -586,6 +587,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -979,6 +981,7 @@ github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfY github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= diff --git a/integration_test.go b/integration_test.go index d8f7133..413e8e2 100644 --- a/integration_test.go +++ b/integration_test.go @@ -35,10 +35,21 @@ type putResponse struct { OID string `json:"object_id"` } +const ( + testContainerName = "friendly" + versionWithNativeNames = "0.27.5" +) + func TestIntegration(t *testing.T) { rootCtx := context.Background() aioImage := "nspccdev/neofs-aio-testcontainer:" - versions := []string{"0.24.0", "0.25.1", "0.26.1", "0.27.0", "latest"} + versions := []string{ + "0.24.0", + "0.25.1", + "0.26.1", + "0.27.5", + "latest", + } key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) @@ -48,13 +59,13 @@ func TestIntegration(t *testing.T) { aioContainer := createDockerContainer(ctx, t, aioImage+version) cancel := runServer() clientPool := getPool(ctx, t, key) - CID, err := createContainer(ctx, t, clientPool) + CID, err := createContainer(ctx, t, clientPool, version) require.NoError(t, err, version) - t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) }) - t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID) }) - t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID) }) - t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID) }) + t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID, version) }) + t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID, version) }) + t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID, version) }) + t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID, version) }) cancel() err = aioContainer.Terminate(ctx) @@ -74,10 +85,19 @@ func runServer() context.CancelFunc { return cancel } -func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { +func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID *cid.ID, version string) { + url := "http://localhost:8082/upload/" + CID.String() + makePutRequestAndCheck(ctx, t, p, CID, url) + + if version >= versionWithNativeNames { + url = "http://localhost:8082/upload/" + testContainerName + makePutRequestAndCheck(ctx, t, p, CID, url) + } +} + +func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.ID, url string) { content := "content of file" keyAttr, valAttr := "User-Attribute", "user value" - attributes := map[string]string{ object.AttributeFileName: "newFile.txt", keyAttr: valAttr, @@ -92,23 +112,32 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci err = w.Close() require.NoError(t, err) - request, err := http.NewRequest(http.MethodPost, "http://localhost:8082/upload/"+CID.String(), &buff) + request, err := http.NewRequest(http.MethodPost, url, &buff) require.NoError(t, err) request.Header.Set("Content-Type", w.FormDataContentType()) request.Header.Set("X-Attribute-"+keyAttr, valAttr) resp, err := http.DefaultClient.Do(request) require.NoError(t, err) + defer func() { - err = resp.Body.Close() + err := resp.Body.Close() require.NoError(t, err) }() - addr := &putResponse{} - err = json.NewDecoder(resp.Body).Decode(addr) + body, err := io.ReadAll(resp.Body) require.NoError(t, err) - err = CID.DecodeString(addr.CID) + if resp.StatusCode != http.StatusOK { + fmt.Println(string(body)) + } + require.Equal(t, http.StatusOK, resp.StatusCode) + + addr := &putResponse{} + err = json.Unmarshal(body, addr) + require.NoError(t, err) + + err = cnrID.DecodeString(addr.CID) require.NoError(t, err) id := new(oid.ID) @@ -116,7 +145,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci require.NoError(t, err) objectAddress := address.NewAddress() - objectAddress.SetContainerID(*CID) + objectAddress.SetContainerID(*cnrID) objectAddress.SetObjectID(*id) payload := bytes.NewBuffer(nil) @@ -124,7 +153,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci var prm pool.PrmObjectGet prm.SetAddress(*objectAddress) - res, err := clientPool.GetObject(ctx, prm) + res, err := p.GetObject(ctx, prm) require.NoError(t, err) _, err = io.Copy(payload, res.Payload) @@ -137,7 +166,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci } } -func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { +func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) { content := "content of file" attributes := map[string]string{ "some-attr": "some-get-value", @@ -147,8 +176,18 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + id.String()) require.NoError(t, err) + checkGetResponse(t, resp, content, attributes) + + if version >= versionWithNativeNames { + resp, err = http.Get("http://localhost:8082/get/" + testContainerName + "/" + id.String()) + require.NoError(t, err) + checkGetResponse(t, resp, content, attributes) + } +} + +func checkGetResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) { defer func() { - err = resp.Body.Close() + err := resp.Body.Close() require.NoError(t, err) }() @@ -161,7 +200,7 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci } } -func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { +func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) { keyAttr, valAttr := "some-attr", "some-get-by-attr-value" content := "content of file" attributes := map[string]string{keyAttr: valAttr} @@ -176,8 +215,18 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci resp, err := http.Get("http://localhost:8082/get_by_attribute/" + CID.String() + "/" + keyAttr + "/" + valAttr) require.NoError(t, err) + checkGetByAttrResponse(t, resp, content, expectedAttr) + + if version >= versionWithNativeNames { + resp, err = http.Get("http://localhost:8082/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr) + require.NoError(t, err) + checkGetByAttrResponse(t, resp, content, expectedAttr) + } +} + +func checkGetByAttrResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) { defer func() { - err = resp.Body.Close() + err := resp.Body.Close() require.NoError(t, err) }() @@ -185,12 +234,12 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci require.NoError(t, err) require.Equal(t, content, string(data)) - for k, v := range expectedAttr { + for k, v := range attributes { require.Equal(t, v, resp.Header.Get(k)) } } -func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { +func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) { names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"} contents := []string{"content of file1", "content of file2"} attributes1 := map[string]string{attributeFilePath: names[0]} @@ -199,28 +248,35 @@ func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.I putObject(ctx, t, clientPool, CID, contents[0], attributes1) putObject(ctx, t, clientPool, CID, contents[1], attributes2) - resp, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder") + baseURL := "http://localhost:8082/zip/" + CID.String() + makeZipTest(t, baseURL, names, contents) + + if version >= versionWithNativeNames { + baseURL = "http://localhost:8082/zip/" + testContainerName + makeZipTest(t, baseURL, names, contents) + } +} + +func makeZipTest(t *testing.T, baseURL string, names, contents []string) { + url := baseURL + "/zipfolder" + makeZipRequest(t, url, names, contents) + + // check nested folder + url = baseURL + "/zipfolder/dir" + makeZipRequest(t, url, names[:1], contents[:1]) +} + +func makeZipRequest(t *testing.T, url string, names, contents []string) { + resp, err := http.Get(url) require.NoError(t, err) defer func() { - err = resp.Body.Close() + err := resp.Body.Close() require.NoError(t, err) }() data, err := io.ReadAll(resp.Body) require.NoError(t, err) checkZip(t, data, resp.ContentLength, names, contents) - - // check nested folder - resp2, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder/dir") - require.NoError(t, err) - defer func() { - err = resp2.Body.Close() - require.NoError(t, err) - }() - - data2, err := io.ReadAll(resp2.Body) - require.NoError(t, err) - checkZip(t, data2, resp2.ContentLength, names[:1], contents[:1]) } func checkZip(t *testing.T, data []byte, length int64, names, contents []string) { @@ -273,6 +329,8 @@ func getDefaultConfig() *viper.Viper { v.SetDefault(cfgPeers+".0.weight", 1) v.SetDefault(cfgPeers+".0.priority", 1) + v.SetDefault(cfgRPCEndpoint, "http://127.0.0.1:30333") + return v } @@ -290,17 +348,20 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool return clientPool } -func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool) (*cid.ID, error) { +func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, version string) (*cid.ID, error) { pp, err := policy.Parse("REP 1") require.NoError(t, err) cnr := container.New( container.WithPolicy(pp), container.WithCustomBasicACL(0x0FFFFFFF), - container.WithAttribute(container.AttributeName, "friendlyName"), container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10))) cnr.SetOwnerID(clientPool.OwnerID()) + if version >= versionWithNativeNames { + container.SetNativeName(cnr, testContainerName) + } + var waitPrm pool.WaitParams waitPrm.SetTimeout(15 * time.Second) waitPrm.SetPollInterval(3 * time.Second) diff --git a/resolver/neofs.go b/resolver/neofs.go new file mode 100644 index 0000000..caa460b --- /dev/null +++ b/resolver/neofs.go @@ -0,0 +1,46 @@ +package resolver + +import ( + "context" + "errors" + "fmt" + + "github.com/nspcc-dev/neofs-sdk-go/netmap" + "github.com/nspcc-dev/neofs-sdk-go/pool" +) + +// NeoFSResolver represents virtual connection to the NeoFS network. +// It implements resolver.NeoFS. +type NeoFSResolver struct { + pool *pool.Pool +} + +// NewNeoFSResolver creates new NeoFSResolver using provided pool.Pool. +func NewNeoFSResolver(p *pool.Pool) *NeoFSResolver { + return &NeoFSResolver{pool: p} +} + +// SystemDNS implements resolver.NeoFS interface method. +func (x *NeoFSResolver) SystemDNS(ctx context.Context) (string, error) { + networkInfo, err := x.pool.NetworkInfo(ctx) + if err != nil { + return "", fmt.Errorf("read network info via client: %w", err) + } + + var domain string + + networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool { + if string(parameter.Key()) == "SystemDNS" { + domain = string(parameter.Value()) + return true + } + + return false + }) + + if domain == "" { + return "", errors.New("system DNS parameter not found or empty") + } + + return domain, nil +} diff --git a/resolver/resolver.go b/resolver/resolver.go new file mode 100644 index 0000000..9ea9eb8 --- /dev/null +++ b/resolver/resolver.go @@ -0,0 +1,140 @@ +package resolver + +import ( + "context" + "fmt" + + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/nspcc-dev/neofs-sdk-go/ns" +) + +const ( + NNSResolver = "nns" + DNSResolver = "dns" +) + +// NeoFS represents virtual connection to the NeoFS network. +type NeoFS interface { + // SystemDNS reads system DNS network parameters of the NeoFS. + // + // Returns exactly on non-zero value. Returns any error encountered + // which prevented the parameter to be read. + SystemDNS(context.Context) (string, error) +} + +type Config struct { + NeoFS NeoFS + RPCAddress string +} + +type ContainerResolver struct { + Name string + resolve func(context.Context, string) (*cid.ID, error) + + next *ContainerResolver +} + +func (r *ContainerResolver) SetResolveFunc(fn func(context.Context, string) (*cid.ID, error)) { + r.resolve = fn +} + +func (r *ContainerResolver) Resolve(ctx context.Context, name string) (*cid.ID, error) { + cnrID, err := r.resolve(ctx, name) + if err != nil { + if r.next != nil { + cnrID, inErr := r.next.Resolve(ctx, name) + if inErr != nil { + return nil, fmt.Errorf("%s; %w", err.Error(), inErr) + } + return cnrID, nil + } + return nil, err + } + return cnrID, nil +} + +func NewResolver(order []string, cfg *Config) (*ContainerResolver, error) { + if len(order) == 0 { + return nil, fmt.Errorf("resolving order must not be empty") + } + + bucketResolver, err := newResolver(order[len(order)-1], cfg, nil) + if err != nil { + return nil, err + } + + for i := len(order) - 2; i >= 0; i-- { + resolverName := order[i] + next := bucketResolver + + bucketResolver, err = newResolver(resolverName, cfg, next) + if err != nil { + return nil, err + } + } + + return bucketResolver, nil +} + +func newResolver(name string, cfg *Config, next *ContainerResolver) (*ContainerResolver, error) { + switch name { + case DNSResolver: + return NewDNSResolver(cfg.NeoFS, next) + case NNSResolver: + return NewNNSResolver(cfg.RPCAddress, next) + default: + return nil, fmt.Errorf("unknown resolver: %s", name) + } +} + +func NewDNSResolver(neoFS NeoFS, next *ContainerResolver) (*ContainerResolver, error) { + if neoFS == nil { + return nil, fmt.Errorf("pool must not be nil for DNS resolver") + } + + var dns ns.DNS + + resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) { + domain, err := neoFS.SystemDNS(ctx) + if err != nil { + return nil, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err) + } + + domain = name + "." + domain + cnrID, err := dns.ResolveContainerName(domain) + if err != nil { + return nil, fmt.Errorf("couldn't resolve container '%s' as '%s': %w", name, domain, err) + } + return &cnrID, nil + } + + return &ContainerResolver{ + Name: DNSResolver, + + resolve: resolveFunc, + next: next, + }, nil +} + +func NewNNSResolver(rpcAddress string, next *ContainerResolver) (*ContainerResolver, error) { + var nns ns.NNS + + if err := nns.Dial(rpcAddress); err != nil { + return nil, fmt.Errorf("could not dial nns: %w", err) + } + + resolveFunc := func(_ context.Context, name string) (*cid.ID, error) { + cnrID, err := nns.ResolveContainerName(name) + if err != nil { + return nil, fmt.Errorf("couldn't resolve container '%s': %w", name, err) + } + return &cnrID, nil + } + + return &ContainerResolver{ + Name: NNSResolver, + + resolve: resolveFunc, + next: next, + }, nil +} diff --git a/settings.go b/settings.go index 62af367..7a0774f 100644 --- a/settings.go +++ b/settings.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/nspcc-dev/neofs-http-gw/resolver" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/valyala/fasthttp" @@ -49,6 +50,12 @@ const ( // Peers. cfgPeers = "peers" + // NeoGo. + cfgRPCEndpoint = "rpc_endpoint" + + // Resolving. + cfgResolveOrder = "resolve_order" + // Zip compression. cfgZipCompression = "zip.compression" @@ -99,6 +106,8 @@ func settings() *viper.Viper { flags.String(cfgTLSKey, "", "TLS key path") peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") + resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order") + // set defaults: // logger: @@ -126,6 +135,10 @@ func settings() *viper.Viper { panic(err) } + if resolveMethods != nil { + v.SetDefault(cfgResolveOrder, *resolveMethods) + } + switch { case help != nil && *help: fmt.Printf("NeoFS HTTP Gateway %s\n", Version) diff --git a/uploader/upload.go b/uploader/upload.go index 3910eac..d60b62f 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -9,11 +9,11 @@ import ( "strconv" "time" + "github.com/nspcc-dev/neofs-http-gw/resolver" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/bearer" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object/address" @@ -35,6 +35,7 @@ type Uploader struct { log *zap.Logger pool *pool.Pool enableDefaultTimestamp bool + containerResolver *resolver.ContainerResolver } type epochDurations struct { @@ -45,33 +46,41 @@ type epochDurations struct { // New creates a new Uploader using specified logger, connection pool and // other options. -func New(ctx context.Context, log *zap.Logger, conns *pool.Pool, enableDefaultTimestamp bool) *Uploader { - return &Uploader{ctx, log, conns, enableDefaultTimestamp} +func New(ctx context.Context, params *utils.AppParams, enableDefaultTimestamp bool) *Uploader { + return &Uploader{ + appCtx: ctx, + log: params.Logger, + pool: params.Pool, + enableDefaultTimestamp: enableDefaultTimestamp, + containerResolver: params.Resolver, + } } // Upload handles multipart upload request. func (u *Uploader) Upload(c *fasthttp.RequestCtx) { var ( - err error file MultipartFile idObj *oid.ID addr = address.NewAddress() - idCnr = new(cid.ID) scid, _ = c.UserValue("cid").(string) log = u.log.With(zap.String("cid", scid)) bodyStream = c.RequestBodyStream() drainBuf = make([]byte, drainBufSize) ) - if err = tokens.StoreBearerToken(c); err != nil { + + if err := tokens.StoreBearerToken(c); err != nil { log.Error("could not fetch bearer token", zap.Error(err)) response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest) return } - if err = idCnr.DecodeString(scid); err != nil { + + idCnr, err := utils.GetContainerID(u.appCtx, scid, u.containerResolver) + if err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } + defer func() { // If the temporary reader can be closed - let's close it. if file == nil { diff --git a/utils/params.go b/utils/params.go new file mode 100644 index 0000000..4bcd352 --- /dev/null +++ b/utils/params.go @@ -0,0 +1,13 @@ +package utils + +import ( + "github.com/nspcc-dev/neofs-http-gw/resolver" + "github.com/nspcc-dev/neofs-sdk-go/pool" + "go.uber.org/zap" +) + +type AppParams struct { + Logger *zap.Logger + Pool *pool.Pool + Resolver *resolver.ContainerResolver +} diff --git a/utils/util.go b/utils/util.go new file mode 100644 index 0000000..73068f1 --- /dev/null +++ b/utils/util.go @@ -0,0 +1,21 @@ +package utils + +import ( + "context" + + "github.com/nspcc-dev/neofs-http-gw/resolver" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" +) + +// GetContainerID decode container id, if it's not a valid container id +// then trey to resolve name using provided resolver. +func GetContainerID(ctx context.Context, containerID string, resolver *resolver.ContainerResolver) (*cid.ID, error) { + cnrID := new(cid.ID) + err := cnrID.DecodeString(containerID) + if err != nil { + if resolver != nil { + cnrID, err = resolver.Resolve(ctx, containerID) + } + } + return cnrID, err +} From 97cde6e2628bf8c6f6df7bf0895d67be2fb29b98 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 28 Apr 2022 17:57:29 +0300 Subject: [PATCH 263/548] Release v0.20.0 Signed-off-by: Denis Kirillov --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e3b1b8..a92d3d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ This document outlines major changes between releases. +## [0.20.0] - 2022-04-29 + +### Fixed +- Get rid of data race on server shutdown (#145) +- Improved English in docs and comments (#153) +- Use `FilePath` to download zip (#150) + +### Added +- Support container name NNS resolving (#142) + +### Changed +- Updated docs (#133, #140) +- Increased default read/write timeouts (#154) +- Updated SDK (#137, #139) +- Updated go version to 1.17 (#143) +- Improved error messages (#144) + ## [0.19.0] - 2022-03-16 ### Fixed @@ -146,3 +163,4 @@ releases. [0.17.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.16.1...v0.17.0 [0.18.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.17.0...v0.18.0 [0.19.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.18.0...v0.19.0 +[0.20.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.19.0...v0.20.0 From 0bdc72cb28f656528603913c690865755607fd94 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 3 Jun 2022 17:00:58 +0300 Subject: [PATCH 264/548] [#161] Fix lint Signed-off-by: Denis Kirillov --- Makefile | 2 +- tokens/bearer-token.go | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f152056..594ae55 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ docker/lint: -v `pwd`:/src \ -u `stat -c "%u:%g" .` \ --env HOME=/src \ - golangci/golangci-lint:v1.39 bash -c 'cd /src/ && make lint' + golangci/golangci-lint:v1.46.2 bash -c 'cd /src/ && make lint' # Print version version: diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index 672138d..cf7dc45 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -89,8 +89,6 @@ func fetchBearerToken(ctx *fasthttp.RequestCtx) (*bearer.Token, error) { } else if err = tkn.Unmarshal(data); err != nil { lastErr = fmt.Errorf("can't unmarshal bearer token: %w", err) continue - } else if tkn == nil { - continue } return tkn, nil From 8ced5f40e8a249e2f031a9e473f2ae362c5af168 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 3 Jun 2022 16:56:09 +0300 Subject: [PATCH 265/548] [#158] Update CodeQL action to v2 Signed-off-by: Denis Kirillov --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 942e14d..5a107f1 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -50,7 +50,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -64,4 +64,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 From 562c7e994c62db17d4c4319e12167d579102f559 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 29 Apr 2022 18:25:26 +0300 Subject: [PATCH 266/548] [#148] Add custom multipart reader Signed-off-by: Denis Kirillov --- uploader/multipart.go | 4 +- uploader/multipart/multipart.go | 422 ++++++++++++++++++++++++++++++++ uploader/multipart_test.go | 161 ++++++++++++ 3 files changed, 586 insertions(+), 1 deletion(-) create mode 100644 uploader/multipart/multipart.go create mode 100644 uploader/multipart_test.go diff --git a/uploader/multipart.go b/uploader/multipart.go index c79ab94..5928e60 100644 --- a/uploader/multipart.go +++ b/uploader/multipart.go @@ -2,8 +2,8 @@ package uploader import ( "io" - "mime/multipart" + "github.com/nspcc-dev/neofs-http-gw/uploader/multipart" "go.uber.org/zap" ) @@ -15,6 +15,8 @@ type MultipartFile interface { } func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartFile, error) { + // To have a custom buffer (3mb) the custom multipart reader is used. + // https://github.com/nspcc-dev/neofs-http-gw/issues/148 reader := multipart.NewReader(r, boundary) for { diff --git a/uploader/multipart/multipart.go b/uploader/multipart/multipart.go new file mode 100644 index 0000000..6fc6031 --- /dev/null +++ b/uploader/multipart/multipart.go @@ -0,0 +1,422 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// + +/* +Package multipart implements MIME multipart parsing, as defined in RFC +2046. + +The implementation is sufficient for HTTP (RFC 2388) and the multipart +bodies generated by popular browsers. +*/ +package multipart + +import ( + "bufio" + "bytes" + "fmt" + "io" + "mime" + "mime/quotedprintable" + "net/textproto" + "strings" +) + +var emptyParams = make(map[string]string) + +// This constant needs to be at least 76 for this package to work correctly. +// This is because \r\n--separator_of_len_70- would fill the buffer and it +// wouldn't be safe to consume a single byte from it. +// This constant is different from the constant in stdlib. The standard value is 4096. +const peekBufferSize = 3 << 20 + +// A Part represents a single part in a multipart body. +type Part struct { + // The headers of the body, if any, with the keys canonicalized + // in the same fashion that the Go http.Request headers are. + // For example, "foo-bar" changes case to "Foo-Bar" + Header textproto.MIMEHeader + + mr *Reader + + disposition string + dispositionParams map[string]string + + // r is either a reader directly reading from mr, or it's a + // wrapper around such a reader, decoding the + // Content-Transfer-Encoding + r io.Reader + + n int // known data bytes waiting in mr.bufReader + total int64 // total data bytes read already + err error // error to return when n == 0 + readErr error // read error observed from mr.bufReader +} + +// FormName returns the name parameter if p has a Content-Disposition +// of type "form-data". Otherwise it returns the empty string. +func (p *Part) FormName() string { + // See https://tools.ietf.org/html/rfc2183 section 2 for EBNF + // of Content-Disposition value format. + if p.dispositionParams == nil { + p.parseContentDisposition() + } + if p.disposition != "form-data" { + return "" + } + return p.dispositionParams["name"] +} + +// FileName returns the filename parameter of the Part's +// Content-Disposition header. +func (p *Part) FileName() string { + if p.dispositionParams == nil { + p.parseContentDisposition() + } + return p.dispositionParams["filename"] +} + +func (p *Part) parseContentDisposition() { + v := p.Header.Get("Content-Disposition") + var err error + p.disposition, p.dispositionParams, err = mime.ParseMediaType(v) + if err != nil { + p.dispositionParams = emptyParams + } +} + +// NewReader creates a new multipart Reader reading from r using the +// given MIME boundary. +// +// The boundary is usually obtained from the "boundary" parameter of +// the message's "Content-Type" header. Use mime.ParseMediaType to +// parse such headers. +func NewReader(r io.Reader, boundary string) *Reader { + b := []byte("\r\n--" + boundary + "--") + return &Reader{ + bufReader: bufio.NewReaderSize(&stickyErrorReader{r: r}, peekBufferSize), + nl: b[:2], + nlDashBoundary: b[:len(b)-2], + dashBoundaryDash: b[2:], + dashBoundary: b[2 : len(b)-2], + } +} + +// stickyErrorReader is an io.Reader which never calls Read on its +// underlying Reader once an error has been seen. (the io.Reader +// interface's contract promises nothing about the return values of +// Read calls after an error, yet this package does do multiple Reads +// after error) +type stickyErrorReader struct { + r io.Reader + err error +} + +func (r *stickyErrorReader) Read(p []byte) (n int, _ error) { + if r.err != nil { + return 0, r.err + } + n, r.err = r.r.Read(p) + return n, r.err +} + +func newPart(mr *Reader, rawPart bool) (*Part, error) { + bp := &Part{ + Header: make(map[string][]string), + mr: mr, + } + if err := bp.populateHeaders(); err != nil { + return nil, err + } + bp.r = partReader{bp} + + // rawPart is used to switch between Part.NextPart and Part.NextRawPart. + if !rawPart { + const cte = "Content-Transfer-Encoding" + if strings.EqualFold(bp.Header.Get(cte), "quoted-printable") { + bp.Header.Del(cte) + bp.r = quotedprintable.NewReader(bp.r) + } + } + return bp, nil +} + +func (bp *Part) populateHeaders() error { + r := textproto.NewReader(bp.mr.bufReader) + header, err := r.ReadMIMEHeader() + if err == nil { + bp.Header = header + } + return err +} + +// Read reads the body of a part, after its headers and before the +// next part (if any) begins. +func (p *Part) Read(d []byte) (n int, err error) { + return p.r.Read(d) +} + +// partReader implements io.Reader by reading raw bytes directly from the +// wrapped *Part, without doing any Transfer-Encoding decoding. +type partReader struct { + p *Part +} + +func (pr partReader) Read(d []byte) (int, error) { + p := pr.p + br := p.mr.bufReader + + // Read into buffer until we identify some data to return, + // or we find a reason to stop (boundary or read error). + for p.n == 0 && p.err == nil { + peek, _ := br.Peek(br.Buffered()) + p.n, p.err = scanUntilBoundary(peek, p.mr.dashBoundary, p.mr.nlDashBoundary, p.total, p.readErr) + if p.n == 0 && p.err == nil { + // Force buffered I/O to read more into buffer. + _, p.readErr = br.Peek(len(peek) + 1) + if p.readErr == io.EOF { + p.readErr = io.ErrUnexpectedEOF + } + } + } + + // Read out from "data to return" part of buffer. + if p.n == 0 { + return 0, p.err + } + n := len(d) + if n > p.n { + n = p.n + } + n, _ = br.Read(d[:n]) + p.total += int64(n) + p.n -= n + if p.n == 0 { + return n, p.err + } + return n, nil +} + +// scanUntilBoundary scans buf to identify how much of it can be safely +// returned as part of the Part body. +// dashBoundary is "--boundary". +// nlDashBoundary is "\r\n--boundary" or "\n--boundary", depending on what mode we are in. +// The comments below (and the name) assume "\n--boundary", but either is accepted. +// total is the number of bytes read out so far. If total == 0, then a leading "--boundary" is recognized. +// readErr is the read error, if any, that followed reading the bytes in buf. +// scanUntilBoundary returns the number of data bytes from buf that can be +// returned as part of the Part body and also the error to return (if any) +// once those data bytes are done. +func scanUntilBoundary(buf, dashBoundary, nlDashBoundary []byte, total int64, readErr error) (int, error) { + if total == 0 { + // At beginning of body, allow dashBoundary. + if bytes.HasPrefix(buf, dashBoundary) { + switch matchAfterPrefix(buf, dashBoundary, readErr) { + case -1: + return len(dashBoundary), nil + case 0: + return 0, nil + case +1: + return 0, io.EOF + } + } + if bytes.HasPrefix(dashBoundary, buf) { + return 0, readErr + } + } + + // Search for "\n--boundary". + if i := bytes.Index(buf, nlDashBoundary); i >= 0 { + switch matchAfterPrefix(buf[i:], nlDashBoundary, readErr) { + case -1: + return i + len(nlDashBoundary), nil + case 0: + return i, nil + case +1: + return i, io.EOF + } + } + if bytes.HasPrefix(nlDashBoundary, buf) { + return 0, readErr + } + + // Otherwise, anything up to the final \n is not part of the boundary + // and so must be part of the body. + // Also if the section from the final \n onward is not a prefix of the boundary, + // it too must be part of the body. + i := bytes.LastIndexByte(buf, nlDashBoundary[0]) + if i >= 0 && bytes.HasPrefix(nlDashBoundary, buf[i:]) { + return i, nil + } + return len(buf), readErr +} + +// matchAfterPrefix checks whether buf should be considered to match the boundary. +// The prefix is "--boundary" or "\r\n--boundary" or "\n--boundary", +// and the caller has verified already that bytes.HasPrefix(buf, prefix) is true. +// +// matchAfterPrefix returns +1 if the buffer does match the boundary, +// meaning the prefix is followed by a dash, space, tab, cr, nl, or end of input. +// It returns -1 if the buffer definitely does NOT match the boundary, +// meaning the prefix is followed by some other character. +// For example, "--foobar" does not match "--foo". +// It returns 0 more input needs to be read to make the decision, +// meaning that len(buf) == len(prefix) and readErr == nil. +func matchAfterPrefix(buf, prefix []byte, readErr error) int { + if len(buf) == len(prefix) { + if readErr != nil { + return +1 + } + return 0 + } + c := buf[len(prefix)] + if c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '-' { + return +1 + } + return -1 +} + +func (p *Part) Close() error { + io.Copy(io.Discard, p) + return nil +} + +// Reader is an iterator over parts in a MIME multipart body. +// Reader's underlying parser consumes its input as needed. Seeking +// isn't supported. +type Reader struct { + bufReader *bufio.Reader + + currentPart *Part + partsRead int + + nl []byte // "\r\n" or "\n" (set after seeing first boundary line) + nlDashBoundary []byte // nl + "--boundary" + dashBoundaryDash []byte // "--boundary--" + dashBoundary []byte // "--boundary" +} + +// NextPart returns the next part in the multipart or an error. +// When there are no more parts, the error io.EOF is returned. +// +// As a special case, if the "Content-Transfer-Encoding" header +// has a value of "quoted-printable", that header is instead +// hidden and the body is transparently decoded during Read calls. +func (r *Reader) NextPart() (*Part, error) { + return r.nextPart(false) +} + +// NextRawPart returns the next part in the multipart or an error. +// When there are no more parts, the error io.EOF is returned. +// +// Unlike NextPart, it does not have special handling for +// "Content-Transfer-Encoding: quoted-printable". +func (r *Reader) NextRawPart() (*Part, error) { + return r.nextPart(true) +} + +func (r *Reader) nextPart(rawPart bool) (*Part, error) { + if r.currentPart != nil { + r.currentPart.Close() + } + if string(r.dashBoundary) == "--" { + return nil, fmt.Errorf("multipart: boundary is empty") + } + expectNewPart := false + for { + line, err := r.bufReader.ReadSlice('\n') + + if err == io.EOF && r.isFinalBoundary(line) { + // If the buffer ends in "--boundary--" without the + // trailing "\r\n", ReadSlice will return an error + // (since it's missing the '\n'), but this is a valid + // multipart EOF so we need to return io.EOF instead of + // a fmt-wrapped one. + return nil, io.EOF + } + if err != nil { + return nil, fmt.Errorf("multipart: NextPart: %v", err) + } + + if r.isBoundaryDelimiterLine(line) { + r.partsRead++ + bp, err := newPart(r, rawPart) + if err != nil { + return nil, err + } + r.currentPart = bp + return bp, nil + } + + if r.isFinalBoundary(line) { + // Expected EOF + return nil, io.EOF + } + + if expectNewPart { + return nil, fmt.Errorf("multipart: expecting a new Part; got line %q", string(line)) + } + + if r.partsRead == 0 { + // skip line + continue + } + + // Consume the "\n" or "\r\n" separator between the + // body of the previous part and the boundary line we + // now expect will follow. (either a new part or the + // end boundary) + if bytes.Equal(line, r.nl) { + expectNewPart = true + continue + } + + return nil, fmt.Errorf("multipart: unexpected line in Next(): %q", line) + } +} + +// isFinalBoundary reports whether line is the final boundary line +// indicating that all parts are over. +// It matches `^--boundary--[ \t]*(\r\n)?$` +func (mr *Reader) isFinalBoundary(line []byte) bool { + if !bytes.HasPrefix(line, mr.dashBoundaryDash) { + return false + } + rest := line[len(mr.dashBoundaryDash):] + rest = skipLWSPChar(rest) + return len(rest) == 0 || bytes.Equal(rest, mr.nl) +} + +func (mr *Reader) isBoundaryDelimiterLine(line []byte) (ret bool) { + // https://tools.ietf.org/html/rfc2046#section-5.1 + // The boundary delimiter line is then defined as a line + // consisting entirely of two hyphen characters ("-", + // decimal value 45) followed by the boundary parameter + // value from the Content-Type header field, optional linear + // whitespace, and a terminating CRLF. + if !bytes.HasPrefix(line, mr.dashBoundary) { + return false + } + rest := line[len(mr.dashBoundary):] + rest = skipLWSPChar(rest) + + // On the first part, see our lines are ending in \n instead of \r\n + // and switch into that mode if so. This is a violation of the spec, + // but occurs in practice. + if mr.partsRead == 0 && len(rest) == 1 && rest[0] == '\n' { + mr.nl = mr.nl[1:] + mr.nlDashBoundary = mr.nlDashBoundary[1:] + } + return bytes.Equal(rest, mr.nl) +} + +// skipLWSPChar returns b with leading spaces and tabs removed. +// RFC 822 defines: +// LWSP-char = SPACE / HTAB +func skipLWSPChar(b []byte) []byte { + for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') { + b = b[1:] + } + return b +} diff --git a/uploader/multipart_test.go b/uploader/multipart_test.go new file mode 100644 index 0000000..9763f88 --- /dev/null +++ b/uploader/multipart_test.go @@ -0,0 +1,161 @@ +package uploader + +import ( + "crypto/rand" + "fmt" + "io" + "mime/multipart" + "os" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func generateRandomFile(size int64) (string, error) { + file, err := os.CreateTemp("", "data") + if err != nil { + return "", err + } + + _, err = io.CopyN(file, rand.Reader, size) + if err != nil { + return "", err + } + + return file.Name(), file.Close() +} + +func BenchmarkAll(b *testing.B) { + fileName, err := generateRandomFile(1024 * 1024 * 256) + require.NoError(b, err) + fmt.Println(fileName) + defer os.Remove(fileName) + + b.Run("bare", func(b *testing.B) { + for i := 0; i < b.N; i++ { + err := bareRead(fileName) + require.NoError(b, err) + } + }) + + b.Run("default", func(b *testing.B) { + for i := 0; i < b.N; i++ { + err := defaultMultipart(fileName) + require.NoError(b, err) + } + }) + + b.Run("custom", func(b *testing.B) { + for i := 0; i < b.N; i++ { + err := customMultipart(fileName) + require.NoError(b, err) + } + }) +} + +func defaultMultipart(filename string) error { + r, bound := multipartFile(filename) + + logger, err := zap.NewProduction() + if err != nil { + return err + } + + file, err := fetchMultipartFileDefault(logger, r, bound) + if err != nil { + return err + } + + _, err = io.Copy(io.Discard, file) + return err +} + +func TestName(t *testing.T) { + fileName, err := generateRandomFile(1024 * 1024 * 256) + require.NoError(t, err) + fmt.Println(fileName) + defer os.Remove(fileName) + + err = defaultMultipart(fileName) + require.NoError(t, err) +} + +func customMultipart(filename string) error { + r, bound := multipartFile(filename) + + logger, err := zap.NewProduction() + if err != nil { + return err + } + + file, err := fetchMultipartFile(logger, r, bound) + if err != nil { + return err + } + + _, err = io.Copy(io.Discard, file) + return err +} + +func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (MultipartFile, error) { + reader := multipart.NewReader(r, boundary) + + for { + part, err := reader.NextPart() + if err != nil { + return nil, err + } + + name := part.FormName() + if name == "" { + l.Debug("ignore part, empty form name") + continue + } + + filename := part.FileName() + + // ignore multipart/form-data values + if filename == "" { + l.Debug("ignore part, empty filename", zap.String("form", name)) + + continue + } + + return part, nil + } +} + +func bareRead(filename string) error { + r, _ := multipartFile(filename) + + _, err := io.Copy(io.Discard, r) + return err +} + +func multipartFile(filename string) (*io.PipeReader, string) { + r, w := io.Pipe() + m := multipart.NewWriter(w) + go func() { + defer w.Close() + defer m.Close() + part, err := m.CreateFormFile("myFile", "foo.txt") + if err != nil { + fmt.Println(err) + return + } + + file, err := os.Open(filename) + if err != nil { + fmt.Println(err) + return + } + defer file.Close() + if _, err = io.Copy(part, file); err != nil { + fmt.Println(err) + return + } + }() + + return r, m.Boundary() +} From b0b724393ad8e4f187e4f95110c42007b945a954 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 3 Jun 2022 13:35:27 +0300 Subject: [PATCH 267/548] [#148] Fix lint errors Signed-off-by: Denis Kirillov --- uploader/multipart/multipart.go | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/uploader/multipart/multipart.go b/uploader/multipart/multipart.go index 6fc6031..531b83f 100644 --- a/uploader/multipart/multipart.go +++ b/uploader/multipart/multipart.go @@ -107,7 +107,7 @@ func NewReader(r io.Reader, boundary string) *Reader { // underlying Reader once an error has been seen. (the io.Reader // interface's contract promises nothing about the return values of // Read calls after an error, yet this package does do multiple Reads -// after error) +// after error). type stickyErrorReader struct { r io.Reader err error @@ -142,11 +142,11 @@ func newPart(mr *Reader, rawPart bool) (*Part, error) { return bp, nil } -func (bp *Part) populateHeaders() error { - r := textproto.NewReader(bp.mr.bufReader) +func (p *Part) populateHeaders() error { + r := textproto.NewReader(p.mr.bufReader) header, err := r.ReadMIMEHeader() if err == nil { - bp.Header = header + p.Header = header } return err } @@ -278,7 +278,7 @@ func matchAfterPrefix(buf, prefix []byte, readErr error) int { } func (p *Part) Close() error { - io.Copy(io.Discard, p) + _, _ = io.Copy(io.Discard, p) return nil } @@ -378,37 +378,37 @@ func (r *Reader) nextPart(rawPart bool) (*Part, error) { // isFinalBoundary reports whether line is the final boundary line // indicating that all parts are over. -// It matches `^--boundary--[ \t]*(\r\n)?$` -func (mr *Reader) isFinalBoundary(line []byte) bool { - if !bytes.HasPrefix(line, mr.dashBoundaryDash) { +// It matches `^--boundary--[ \t]*(\r\n)?$`. +func (r *Reader) isFinalBoundary(line []byte) bool { + if !bytes.HasPrefix(line, r.dashBoundaryDash) { return false } - rest := line[len(mr.dashBoundaryDash):] + rest := line[len(r.dashBoundaryDash):] rest = skipLWSPChar(rest) - return len(rest) == 0 || bytes.Equal(rest, mr.nl) + return len(rest) == 0 || bytes.Equal(rest, r.nl) } -func (mr *Reader) isBoundaryDelimiterLine(line []byte) (ret bool) { +func (r *Reader) isBoundaryDelimiterLine(line []byte) (ret bool) { // https://tools.ietf.org/html/rfc2046#section-5.1 // The boundary delimiter line is then defined as a line // consisting entirely of two hyphen characters ("-", // decimal value 45) followed by the boundary parameter // value from the Content-Type header field, optional linear // whitespace, and a terminating CRLF. - if !bytes.HasPrefix(line, mr.dashBoundary) { + if !bytes.HasPrefix(line, r.dashBoundary) { return false } - rest := line[len(mr.dashBoundary):] + rest := line[len(r.dashBoundary):] rest = skipLWSPChar(rest) // On the first part, see our lines are ending in \n instead of \r\n // and switch into that mode if so. This is a violation of the spec, // but occurs in practice. - if mr.partsRead == 0 && len(rest) == 1 && rest[0] == '\n' { - mr.nl = mr.nl[1:] - mr.nlDashBoundary = mr.nlDashBoundary[1:] + if r.partsRead == 0 && len(rest) == 1 && rest[0] == '\n' { + r.nl = r.nl[1:] + r.nlDashBoundary = r.nlDashBoundary[1:] } - return bytes.Equal(rest, mr.nl) + return bytes.Equal(rest, r.nl) } // skipLWSPChar returns b with leading spaces and tabs removed. From 5baa2774d58a1b978261a146ca512c148803e164 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 7 Jun 2022 19:58:33 +0300 Subject: [PATCH 268/548] [#159] Add docker/$(BINS) target Use this target to build binaries in clean docker environment. Signed-off-by: Alex Vanin --- Makefile | 14 ++++++++++++-- README.md | 5 +++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 594ae55..319b6ec 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --dirty --always) +GO_VERSION ?= 1.17 BUILD ?= $(shell date -u --iso=seconds) HUB_IMAGE ?= nspccdev/neofs-http-gw @@ -10,9 +11,9 @@ HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" # List of binaries to build. For now just one. BINDIR = bin DIRS = $(BINDIR) -BINS = "$(BINDIR)/neofs-http-gw" +BINS = $(BINDIR)/neofs-http-gw -.PHONY: help all dep clean fmts fmt imports test cover lint docker/lint +.PHONY: help all $(BINS) $(DIRS) docker/$(BINS) dep test cover fmts fmt imports image image-push dirty-image lint docker/lint version clean # Make all binaries all: $(BINS) @@ -29,6 +30,15 @@ $(DIRS): @echo "⇒ Ensure dir: $@" @mkdir -p $@ +docker/$(BINS): + @echo "=> Building binary using clean Docker environment" + @docker run --rm -t \ + -v `pwd`:/src \ + -w /src \ + -u "$$(id -u):$$(id -g)" \ + --env HOME=/src \ + golang:$(GO_VERSION) make $(BINS) + # Pull go dependencies dep: @printf "⇒ Download requirements: " diff --git a/README.md b/README.md index 94aa369..cbfa992 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,10 @@ NeoFS HTTP Gateway bridges NeoFS internal protocol and HTTP standard. ```go install github.com/nspcc-dev/neofs-http-gw``` Or you can call `make` to build it from the cloned repository (the binary will -end up in `bin/neofs-http-gw`). +end up in `bin/neofs-http-gw`). To build neofs-http-gw binary in clean docker +environment, call `make docker/bin/neofs-http-gw`. -Notable make targets: +Other notable make targets: ``` dep Check and ensure dependencies From 40e86fbfd6daeb137dcda3acd92842dd7d9a8f13 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 7 Jun 2022 20:10:23 +0300 Subject: [PATCH 269/548] [#159] Tidy Makefile - Remove help target. Help target will always ignore variable targets, it is confusing. - Remove imports target. Go fmt is enough. Signed-off-by: Alex Vanin --- Makefile | 23 +++-------------------- README.md | 2 +- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 319b6ec..5d34aec 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --dirty --always) GO_VERSION ?= 1.17 +LINT_VERSION ?= 1.46.2 BUILD ?= $(shell date -u --iso=seconds) HUB_IMAGE ?= nspccdev/neofs-http-gw @@ -13,7 +14,7 @@ BINDIR = bin DIRS = $(BINDIR) BINS = $(BINDIR)/neofs-http-gw -.PHONY: help all $(BINS) $(DIRS) docker/$(BINS) dep test cover fmts fmt imports image image-push dirty-image lint docker/lint version clean +.PHONY: all $(BINS) $(DIRS) docker/$(BINS) dep test cover fmt image image-push dirty-image lint docker/lint version clean # Make all binaries all: $(BINS) @@ -59,19 +60,11 @@ cover: @go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic @go tool cover -html=coverage.txt -o coverage.html -# Run all code formatters -fmts: fmt imports - # Reformat code fmt: @echo "⇒ Processing gofmt check" @GO111MODULE=on gofmt -s -w ./ -# Reformat imports -imports: - @echo "⇒ Processing goimports check" - @GO111MODULE=on goimports -w ./ - # Build clean Docker image image: @echo "⇒ Build NeoFS HTTP Gateway docker image " @@ -107,22 +100,12 @@ docker/lint: -v `pwd`:/src \ -u `stat -c "%u:%g" .` \ --env HOME=/src \ - golangci/golangci-lint:v1.46.2 bash -c 'cd /src/ && make lint' + golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint' # Print version version: @echo $(VERSION) -# Show this help prompt -help: - @echo ' Usage:' - @echo '' - @echo ' make ' - @echo '' - @echo ' Targets:' - @echo '' - @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u - # Clean up clean: rm -rf vendor diff --git a/README.md b/README.md index cbfa992..cf80699 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Other notable make targets: dep Check and ensure dependencies image Build clean docker image dirty-image Build dirty docker image with host-built binaries -fmts Run all code formatters +fmt Format the code lint Run linters version Show current version ``` From 6684ea1e3ad30bacbcd4cc59e41fdd778df2fe8a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 7 Jun 2022 22:05:57 +0300 Subject: [PATCH 270/548] [#159] Add docker/all target Signed-off-by: Alex Vanin --- Makefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5d34aec..52776f1 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,20 @@ BINDIR = bin DIRS = $(BINDIR) BINS = $(BINDIR)/neofs-http-gw -.PHONY: all $(BINS) $(DIRS) docker/$(BINS) dep test cover fmt image image-push dirty-image lint docker/lint version clean +.PHONY: all docker/all $(BINS) $(DIRS) docker/$(BINS) dep test cover fmt image image-push dirty-image lint docker/lint version clean # Make all binaries all: $(BINS) +docker/all: + @echo "=> Building binary using clean Docker environment" + @docker run --rm -t \ + -v `pwd`:/src \ + -w /src \ + -u "$$(id -u):$$(id -g)" \ + --env HOME=/src \ + golang:$(GO_VERSION) make all + $(BINS): $(DIRS) dep @echo "⇒ Build $@" CGO_ENABLED=0 \ From 0a0af13bea5a2e9b7c4cfac82a3ce89c54182214 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 10 Jun 2022 16:21:31 +0300 Subject: [PATCH 271/548] [#159] Unify docker targets for binary building Signed-off-by: Alex Vanin --- Makefile | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 52776f1..a0a5ba0 100644 --- a/Makefile +++ b/Makefile @@ -14,20 +14,11 @@ BINDIR = bin DIRS = $(BINDIR) BINS = $(BINDIR)/neofs-http-gw -.PHONY: all docker/all $(BINS) $(DIRS) docker/$(BINS) dep test cover fmt image image-push dirty-image lint docker/lint version clean +.PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint version clean # Make all binaries all: $(BINS) -docker/all: - @echo "=> Building binary using clean Docker environment" - @docker run --rm -t \ - -v `pwd`:/src \ - -w /src \ - -u "$$(id -u):$$(id -g)" \ - --env HOME=/src \ - golang:$(GO_VERSION) make all - $(BINS): $(DIRS) dep @echo "⇒ Build $@" CGO_ENABLED=0 \ @@ -40,15 +31,6 @@ $(DIRS): @echo "⇒ Ensure dir: $@" @mkdir -p $@ -docker/$(BINS): - @echo "=> Building binary using clean Docker environment" - @docker run --rm -t \ - -v `pwd`:/src \ - -w /src \ - -u "$$(id -u):$$(id -g)" \ - --env HOME=/src \ - golang:$(GO_VERSION) make $(BINS) - # Pull go dependencies dep: @printf "⇒ Download requirements: " @@ -60,6 +42,17 @@ dep: GO111MODULE=on \ go mod tidy -v && echo OK +docker/%: + $(if $(filter $*,all $(BINS)), \ + @echo "=> Running 'make $*' in clean Docker environment" && \ + docker run --rm -t \ + -v `pwd`:/src \ + -w /src \ + -u `stat -c "%u:%g" .` \ + --env HOME=/src \ + golang:$(GO_VERSION) make $*,\ + @echo "supported docker targets: all $(BINS) lint") + # Run tests test: @go test ./... -cover From 85ee0c44a27c358e0726de5a6d3f6c984d54722c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 17 Jun 2022 12:17:02 +0300 Subject: [PATCH 272/548] [#163] Fix zip streaming Signed-off-by: Denis Kirillov --- downloader/download.go | 145 +++++++++++++++++++++-------------------- integration_test.go | 2 +- 2 files changed, 76 insertions(+), 71 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 5456f91..21726f4 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -2,6 +2,7 @@ package downloader import ( "archive/zip" + "bufio" "bytes" "context" "errors" @@ -383,6 +384,19 @@ func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string return d.pool.SearchObjects(d.appCtx, prm) } +func (d *Downloader) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { + method := zip.Store + if d.settings.ZipCompression { + method = zip.Deflate + } + + return zw.CreateHeader(&zip.FileHeader{ + Name: getZipFilePath(obj), + Method: method, + Modified: time.Now(), + }) +} + // DownloadZipped handles zip by prefix requests. func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { scid, _ := c.UserValue("cid").(string) @@ -409,99 +423,90 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { return } - defer resSearch.Close() - c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") c.Response.SetStatusCode(http.StatusOK) - zipWriter := zip.NewWriter(c) - compression := zip.Store - if d.settings.ZipCompression { - compression = zip.Deflate - } + c.SetBodyStreamWriter(func(w *bufio.Writer) { + defer resSearch.Close() - var ( - addr address.Address - resGet *pool.ResGetObject - w io.Writer - bufZip []byte - ) + zipWriter := zip.NewWriter(w) - addr.SetContainerID(*containerID) + var bufZip []byte + var addr address.Address - btoken := bearerToken(c) - empty := true - called := false + empty := true + called := false + btoken := bearerToken(c) + addr.SetContainerID(*containerID) - errIter := resSearch.Iterate(func(id oid.ID) bool { - called = true + errIter := resSearch.Iterate(func(id oid.ID) bool { + called = true - if empty { - bufZip = make([]byte, 1024) // configure? - } + if empty { + bufZip = make([]byte, 3<<20) // the same as for upload + } + empty = false - empty = false + addr.SetObjectID(id) + if err = d.zipObject(zipWriter, addr, btoken, bufZip); err != nil { + return true + } - addr.SetObjectID(id) - - var prm pool.PrmObjectGet - prm.SetAddress(addr) - if btoken != nil { - prm.UseBearer(*btoken) - } - - resGet, err = d.pool.GetObject(d.appCtx, prm) - if err != nil { - err = fmt.Errorf("get NeoFS object: %v", err) - return true - } - - w, err = zipWriter.CreateHeader(&zip.FileHeader{ - Name: getZipFilePath(&resGet.Header), - Method: compression, - Modified: time.Now(), + return false }) - if err != nil { - err = fmt.Errorf("zip create header: %v", err) - return true + if errIter != nil { + log.Error("iterating over selected objects failed", zap.Error(errIter)) + response.Error(c, "iterating over selected objects: "+errIter.Error(), fasthttp.StatusBadRequest) + return + } else if !called { + log.Error("objects not found") + response.Error(c, "objects not found", fasthttp.StatusNotFound) + return } - _, err = io.CopyBuffer(w, resGet.Payload, bufZip) - if err != nil { - err = fmt.Errorf("copy object payload to zip file: %v", err) - return true + if err == nil { + err = zipWriter.Close() } - _ = resGet.Payload.Close() - - err = zipWriter.Flush() if err != nil { - err = fmt.Errorf("flush zip writer: %v", err) - return true + log.Error("file streaming failure", zap.Error(err)) + response.Error(c, "file streaming failure: "+err.Error(), fasthttp.StatusInternalServerError) + return } - - return false }) - if errIter != nil { - log.Error("iterating over selected objects failed", zap.Error(errIter)) - response.Error(c, "iterating over selected objects: "+errIter.Error(), fasthttp.StatusBadRequest) - return - } else if !called { - log.Error("objects not found") - response.Error(c, "objects not found", fasthttp.StatusNotFound) - return - } - - if err == nil { - err = zipWriter.Close() +} + +func (d *Downloader) zipObject(zipWriter *zip.Writer, addr address.Address, btoken *bearer.Token, bufZip []byte) error { + var prm pool.PrmObjectGet + prm.SetAddress(addr) + if btoken != nil { + prm.UseBearer(*btoken) } + resGet, err := d.pool.GetObject(d.appCtx, prm) if err != nil { - log.Error("file streaming failure", zap.Error(err)) - response.Error(c, "file streaming failure: "+err.Error(), fasthttp.StatusInternalServerError) - return + return fmt.Errorf("get NeoFS object: %v", err) } + + objWriter, err := d.addObjectToZip(zipWriter, &resGet.Header) + if err != nil { + return fmt.Errorf("zip create header: %v", err) + } + + if _, err = io.CopyBuffer(objWriter, resGet.Payload, bufZip); err != nil { + return fmt.Errorf("copy object payload to zip file: %v", err) + } + + if err = resGet.Payload.Close(); err != nil { + return fmt.Errorf("object body close error: %w", err) + } + + if err = zipWriter.Flush(); err != nil { + return fmt.Errorf("flush zip writer: %v", err) + } + + return nil } func getZipFilePath(obj *object.Object) string { diff --git a/integration_test.go b/integration_test.go index 413e8e2..ecb85bf 100644 --- a/integration_test.go +++ b/integration_test.go @@ -276,7 +276,7 @@ func makeZipRequest(t *testing.T, url string, names, contents []string) { data, err := io.ReadAll(resp.Body) require.NoError(t, err) - checkZip(t, data, resp.ContentLength, names, contents) + checkZip(t, data, int64(len(data)), names, contents) } func checkZip(t *testing.T, data []byte, length int64, names, contents []string) { From f88b4634e75374ac992a1d95d7aba30601cd6b67 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 20 Jun 2022 12:56:34 +0300 Subject: [PATCH 273/548] [#165] Drop workflow_dispatch Signed-off-by: Denis Kirillov --- .github/workflows/builds.yml | 1 - .github/workflows/tests.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 1ac51ce..8c89e07 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -7,7 +7,6 @@ on: types: [opened, synchronize] paths-ignore: - '**/*.md' - workflow_dispatch: jobs: build_cli: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f03488d..0874b72 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,6 @@ on: types: [opened, synchronize] paths-ignore: - '**/*.md' - workflow_dispatch: jobs: lint: From 6ce5fe80255ac2085031c57ddd88fbd1fd5b1219 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 20 Jun 2022 12:17:00 +0300 Subject: [PATCH 274/548] Release v0.21.0 Signed-off-by: Denis Kirillov --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a92d3d4..9cb81b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ This document outlines major changes between releases. +## [0.21.0] - 2022-06-20 + +### Fixed +- Downloading ZIP archive using streaming (#163) + +### Added +- New make target to build app in docker (#159) + +### Changed +- Increased buffer size for file uploading (#148) +- Updated linter version to v1.46.2 (#161) +- Updated CodeQL version to v2 (#158) + + ## [0.20.0] - 2022-04-29 ### Fixed @@ -164,3 +178,4 @@ releases. [0.18.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.17.0...v0.18.0 [0.19.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.18.0...v0.19.0 [0.20.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.19.0...v0.20.0 +[0.21.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.20.0...v0.21.0 From d81587f3044f1235f1c174e212395c743f3a37c9 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Wed, 29 Jun 2022 15:14:42 +0400 Subject: [PATCH 275/548] [#166] Add comment about empty password to examples Signed-off-by: Angira Kekteeva --- config/config.env | 2 +- config/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.env b/config/config.env index a6bd656..f4f0f07 100644 --- a/config/config.env +++ b/config/config.env @@ -4,7 +4,7 @@ HTTP_GW_WALLET_PATH=/path/to/wallet.json # Account address. If omitted default one will be used. HTTP_GW_WALLET_ADDRESS=NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP -# Passphrase to decrypt wallet. +# Passphrase to decrypt wallet. If you're using a wallet without a password, place '' here. HTTP_GW_WALLET_PASSPHRASE=pwd # Enable metrics. diff --git a/config/config.yaml b/config/config.yaml index 9cdb7ce..ef77302 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,7 +1,7 @@ wallet: path: /path/to/wallet.json # Path to wallet. address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP # Account address. If omitted default one will be used. - passphrase: pwd # Passphrase to decrypt wallet. + passphrase: pwd # Passphrase to decrypt wallet. If you're using a wallet without a password, place '' here. metrics: true # Enable metrics. pprof: true # Enable pprof. From 18212485961a3399b9823cfc5fd07f4f7fcaa86a Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 13 Jul 2022 13:44:04 +0300 Subject: [PATCH 276/548] [#167] Use localhost instead of 127.0.0.1 Signed-off-by: Denis Kirillov --- integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_test.go b/integration_test.go index ecb85bf..9031cc2 100644 --- a/integration_test.go +++ b/integration_test.go @@ -325,11 +325,11 @@ func createDockerContainer(ctx context.Context, t *testing.T, image string) test func getDefaultConfig() *viper.Viper { v := settings() - v.SetDefault(cfgPeers+".0.address", "127.0.0.1:8080") + v.SetDefault(cfgPeers+".0.address", "localhost:8080") v.SetDefault(cfgPeers+".0.weight", 1) v.SetDefault(cfgPeers+".0.priority", 1) - v.SetDefault(cfgRPCEndpoint, "http://127.0.0.1:30333") + v.SetDefault(cfgRPCEndpoint, "http://localhost:30333") return v } From 99eb87bec7a46cda5dd7fe7791bae6d5d1c99f7c Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 14 Jul 2022 11:49:22 +0300 Subject: [PATCH 277/548] Add credits Signed-off-by: Alex Vanin --- CREDITS.md | 20 ++++++++++++++++++++ README.md | 4 ++++ 2 files changed, 24 insertions(+) create mode 100644 CREDITS.md diff --git a/CREDITS.md b/CREDITS.md new file mode 100644 index 0000000..e019a0b --- /dev/null +++ b/CREDITS.md @@ -0,0 +1,20 @@ +# Credits + +In alphabetical order: + +- Alexey Vanin +- Angira Kekteeva +- Denis Kirillov +- Evgeniy Kulikov +- Pavel Korotkov +- Roman Khimov + +# Contributors + +In chronological order: + +- Anatoly Bogatyrev +- Stanislav Bogatyrev +- Anastasia Prasolova +- Leonard Liubich +- Elizaveta Chichindaeva diff --git a/README.md b/README.md index cf80699..9bb0216 100644 --- a/README.md +++ b/README.md @@ -538,3 +538,7 @@ File **eacl.json**: If enabled, Prometheus metrics are available at `/metrics/` path and Pprof at `/debug/pprof`. + +## Credits + +Please see [CREDITS](CREDITS.md) for details. From 4a2f9d6d6c164de78706980c0325caf52afeb79e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 18 Jul 2022 11:52:30 +0300 Subject: [PATCH 278/548] [#169] Unify version output Signed-off-by: Denis Kirillov --- settings.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/settings.go b/settings.go index 7a0774f..8c6a80c 100644 --- a/settings.go +++ b/settings.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "runtime" "sort" "strconv" "strings" @@ -168,7 +169,7 @@ func settings() *viper.Viper { os.Exit(0) case version != nil && *version: - fmt.Printf("NeoFS HTTP Gateway %s\n", Version) + fmt.Printf("NeoFS HTTP Gateway\nVersion: %s\nGoVersion: %s\n", Version, runtime.Version()) os.Exit(0) } From a2d667acf9482a72b7f348494f27d96d36526057 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 18 Jul 2022 11:56:14 +0300 Subject: [PATCH 279/548] [#169] Update VERSION calculating Signed-off-by: Denis Kirillov --- Makefile | 2 +- VERSION | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 VERSION diff --git a/Makefile b/Makefile index a0a5ba0..a5489d1 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ #!/usr/bin/make -f REPO ?= $(shell go list -m) -VERSION ?= $(shell git describe --tags --dirty --always) +VERSION ?= $(shell git describe --tags --dirty --always 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") GO_VERSION ?= 1.17 LINT_VERSION ?= 1.46.2 BUILD ?= $(shell date -u --iso=seconds) diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..759e855 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +v0.21.0 From d9317f261eb0fe58c64d347cc1c27f427c1344bb Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 19 Jul 2022 17:06:17 +0300 Subject: [PATCH 280/548] [#172] Describe configuration Signed-off-by: Denis Kirillov --- README.md | 10 +-- docs/gate-configuration.md | 158 +++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 docs/gate-configuration.md diff --git a/README.md b/README.md index 9bb0216..f98b936 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,7 @@ $ neofs-http-gw -p 192.168.130.72:8080 -w wallet.json --address NfgHwwTi3wHAS8aF ### Binding and TLS -Gateway binds to `0.0.0.0:8082` by default and you can change that with -`--listen_address` option. +You can make the gateway listen on specific address using the `--listen_address` option. It can also provide TLS interface for its users, just specify paths to the key and certificate files via `--tls_key` and `--tls_certificate` parameters. Note @@ -146,8 +145,7 @@ variable to control this behavior. ### Monitoring and metrics -Pprof and Prometheus are integrated into the gateway, but they are not enabled by -default. To enable them use `--pprof` and `--metrics` flags or +Pprof and Prometheus are integrated into the gateway. To enable them use `--pprof` and `--metrics` flags or `HTTP_GW_PPROF`/`HTTP_GW_METRICS` environment variables. ### Timeouts @@ -168,7 +166,7 @@ The gateway supports downloading files by common prefix (like dir) in zip format using config or `HTTP_GW_ZIP_COMPRESSION=true` environment variable. ### Logging -You can specify logging level (default `info`) using variable: +You can specify logging level using variable: ``` HTTP_GW_LOGGER_LEVEL=debug ``` @@ -180,7 +178,7 @@ It can be specified with `--config` parameter: $ neofs-http-gw --config your-config.yaml ``` -See [config](./config/config.yaml) for example. +See [config](./config/config.yaml) and [defaults](./docs/gate-configuration.md) for example. ## HTTP API provided diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md new file mode 100644 index 0000000..d1ef7e5 --- /dev/null +++ b/docs/gate-configuration.md @@ -0,0 +1,158 @@ +# NeoFS HTTP Gateway configuration file + +This section contains detailed NeoFS HTTP Gateway configuration file description +including default config values and some tips to set up configurable values. + +There are some custom types used for brevity: + +* `duration` -- string consisting of a number and a suffix. Suffix examples include `s` (seconds), `m` (minutes), `ms` ( + milliseconds). + +# Structure + +| Section | Description | +|-----------------|-------------------------------------------------------| +| no section | [General parameters](#general-section) | +| `wallet` | [Wallet configuration](#wallet-section) | +| `peers` | [Nodes configuration](#peers-section) | +| `logger` | [Logger configuration](#logger-section) | +| `web` | [Web configuration](#web-section) | +| `upload-header` | [Upload header configuration](#upload-header-section) | +| `zip` | [ZIP configuration](#zip-section) | + +# General section + +```yaml +listen_address: 0.0.0.0:8082 +tls_certificate: /path/to/tls/cert +tls_key: /path/to/tls/key + +rpc_endpoint: http://node4.neofs:40332 +resolve_order: + - nns + - dns + +metrics: false +pprof: false + +connect_timeout: 5s +request_timeout: 5s +rebalance_timer: 30s +``` + +| Parameter | Type | Default value | Description | +|-------------------|------------|----------------|------------------------------------------------------------------------------------| +| `listen_address` | `string` | `0.0.0.0:8082` | The address that the gateway is listening on. | +| `tls_certificate` | `string` | | Path to the TLS certificate. | +| `tls_key` | `string` | | Path to the TLS key. | +| `rpc_endpoint` | `string` | | The address of the RPC host to which the gateway connects to resolve bucket names. | +| `resolve_order` | `[]string` | `[nns, dns]` | Order of bucket name resolvers to use. | +| `metrics` | `bool` | `false` | Flag to enable and expose the prometheus metrics. | +| `pprof` | `bool` | `false` | Flag to enable the profiler. | +| `connect_timeout` | `duration` | `10s` | Timeout to connect to a node. | +| `request_timeout` | `duration` | `15s` | Timeout to check node health during rebalance. | +| `rebalance_timer` | `duration` | `60s` | Interval to check node health. | + +# `wallet` section + +```yaml +wallet: + path: /path/to/wallet.json + address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP + passphrase: pwd +``` + +| Parameter | Type | Default value | Description | +|--------------|----------|---------------|--------------------------------------------------------------------------| +| `path` | `string` | | Path to the wallet. | +| `address` | `string` | | Account address to get from wallet. If omitted default one will be used. | +| `passphrase` | `string` | | Passphrase to decrypt wallet. | + +# `peers` section + +```yaml +# Nodes configuration +# This configuration makes the gateway use the first node (node1.neofs:8080) +# while it's healthy. Otherwise, gateway uses the second node (node2.neofs:8080) +# for 10% of requests and the third node (node3.neofs:8080) for 90% of requests. +# Until nodes with the same priority level are healthy +# nodes with other priority are not used. +# The lower the value, the higher the priority. +peers: + 0: + address: node1.neofs:8080 + priority: 1 + weight: 1 + 1: + address: node2.neofs:8080 + priority: 2 + weight: 0.1 + 2: + address: node3.neofs:8080 + priority: 2 + weight: 0.9 +``` + +| Parameter | Type | Default value | Description | +|------------|----------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| +| `address` | `string` | | Address of storage node. | +| `priority` | `int` | `1` | It allows to group nodes and don't switch group until all nodes with the same priority will be unhealthy. The lower the value, the higher the priority. | +| `weight` | `float` | `1` | Weight of node in the group with the same priority. Distribute requests to nodes proportionally to these values. | + +# `logger` section + +```yaml +logger: + level: debug +``` + +| Parameter | Type | Default value | Description | +|-----------|----------|---------------|----------------------------------------------------------------------------------------------------| +| `level` | `string` | `debug` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | + + +# `web` section + +```yaml +web: + read_buffer_size: 4096 + write_buffer_size: 4096 + read_timeout: 10m + write_timeout: 5m + stream_request_body: true + max_request_body_size: 4194304 +``` + +| Parameter | Type | Default value | Description | +|-------------------------|------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `read_buffer_size` | `int` | `4096` | Per-connection buffer size for requests' reading. This also limits the maximum header size. | +| `write_buffer_size` | `int` | `4096` | Per-connection buffer size for responses' writing. | +| `read_timeout` | `duration` | `10m` | The amount of time allowed to read the full request including body. The connection's read deadline is reset when the connection opens, or for keep-alive connections after the first byte has been read. | +| `write_timeout` | `duration` | `5m` | The maximum duration before timing out writes of the response. It is reset after the request handler has returned. | +| `stream_request_body` | `bool` | `true` | Enables request body streaming, and calls the handler sooner when given body is larger than the current limit. | +| `max_request_body_size` | `int` | `4194304` | Maximum request body size. The server rejects requests with bodies exceeding this limit. | + + +# `upload-header` section + +```yaml +upload_header: + use_default_timestamp: false +``` + +| Parameter | Type | Default value | Description | +|-------------------------|--------|---------------|-------------------------------------------------------------| +| `use_default_timestamp` | `bool` | `false` | Create timestamp for object if it isn't provided by header. | + + +# `zip` section + +```yaml +zip: + compression: false +``` + +| Parameter | Type | Default value | Description | +|---------------|--------|---------------|--------------------------------------------------------------| +| `compression` | `bool` | `false` | Enable zip compression when download files by common prefix. | + From 3c64a33b2f73da0bf69553fdaaf1a1ee3de25ed2 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 19 Jul 2022 17:06:32 +0300 Subject: [PATCH 281/548] [#172] Change pool defaults params Signed-off-by: Denis Kirillov --- settings.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/settings.go b/settings.go index 8c6a80c..1b0c8b9 100644 --- a/settings.go +++ b/settings.go @@ -16,9 +16,9 @@ import ( ) const ( - defaultRebalanceTimer = 15 * time.Second + defaultRebalanceTimer = 60 * time.Second defaultRequestTimeout = 15 * time.Second - defaultConnectTimeout = 30 * time.Second + defaultConnectTimeout = 10 * time.Second cfgListenAddress = "listen_address" cfgTLSCertificate = "tls_certificate" From dd50c4ed557f11bba495b5685fd759b1d6b36b6a Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 21 Jul 2022 09:21:58 +0300 Subject: [PATCH 282/548] [#172] Change rpc_endpoint param example Signed-off-by: Denis Kirillov --- docs/gate-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index d1ef7e5..0950e5d 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -27,7 +27,7 @@ listen_address: 0.0.0.0:8082 tls_certificate: /path/to/tls/cert tls_key: /path/to/tls/key -rpc_endpoint: http://node4.neofs:40332 +rpc_endpoint: http://morph-chain.neofs.devenv:30333 resolve_order: - nns - dns From cf018c2fab9b6ea3bb6aba52158ce11ffbd8b61f Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 22 Jul 2022 11:30:57 +0300 Subject: [PATCH 283/548] [#175] Add health metric Signed-off-by: Denis Kirillov --- app.go | 13 +++++++++++++ metrics.go | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/app.go b/app.go index 26201c6..edb3d31 100644 --- a/app.go +++ b/app.go @@ -31,6 +31,7 @@ type ( webServer *fasthttp.Server webDone chan struct{} resolver *resolver.ContainerResolver + metrics GateMetricsProvider } // App is an interface for the main gateway function. @@ -41,6 +42,10 @@ type ( // Option is an application option. Option func(a *app) + + GateMetricsProvider interface { + SetHealth(int32) + } ) // WithLogger returns Option to set a specific logger. @@ -151,6 +156,10 @@ func newApp(ctx context.Context, opt ...Option) App { a.log.Info("container resolver is disabled") } + if a.cfg.GetBool(cmdMetrics) { + a.metrics = newGateMetrics() + } + return a } @@ -231,6 +240,10 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*ecds func (a *app) Wait() { a.log.Info("starting application", zap.String("app_name", "neofs-http-gw"), zap.String("version", Version)) + if a.metrics != nil { + a.metrics.SetHealth(1) + } + <-a.webDone // wait for web-server to be stopped } diff --git a/metrics.go b/metrics.go index 40491bc..4836d6b 100644 --- a/metrics.go +++ b/metrics.go @@ -9,6 +9,47 @@ import ( "go.uber.org/zap" ) +const ( + namespace = "neofs_http_gw" + stateSubsystem = "state" +) + +type GateMetrics struct { + stateMetrics +} + +type stateMetrics struct { + healthCheck prometheus.Gauge +} + +func newGateMetrics() *GateMetrics { + stateMetric := newStateMetrics() + stateMetric.register() + + return &GateMetrics{ + stateMetrics: stateMetric, + } +} + +func newStateMetrics() stateMetrics { + return stateMetrics{ + healthCheck: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: stateSubsystem, + Name: "health", + Help: "Current HTTP gateway state", + }), + } +} + +func (m stateMetrics) register() { + prometheus.MustRegister(m.healthCheck) +} + +func (m stateMetrics) SetHealth(s int32) { + m.healthCheck.Set(float64(s)) +} + func attachMetrics(r *router.Router, l *zap.Logger) { r.GET("/metrics/", metricsHandler(prometheus.DefaultGatherer, l)) } From f46ee45843bfc192520fe0463b6ca0d87f391f4a Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 22 Jul 2022 16:48:26 +0300 Subject: [PATCH 284/548] [#175] Return pointer from newStateMetrics Signed-off-by: Denis Kirillov --- metrics.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metrics.go b/metrics.go index 4836d6b..bb92b6a 100644 --- a/metrics.go +++ b/metrics.go @@ -27,12 +27,12 @@ func newGateMetrics() *GateMetrics { stateMetric.register() return &GateMetrics{ - stateMetrics: stateMetric, + stateMetrics: *stateMetric, } } -func newStateMetrics() stateMetrics { - return stateMetrics{ +func newStateMetrics() *stateMetrics { + return &stateMetrics{ healthCheck: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: stateSubsystem, From 0fb75a1cde683bf2872f25e6e80fcc8c654b6b8b Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 25 Jul 2022 12:47:48 +0300 Subject: [PATCH 285/548] [#178] Update SDK to v1.0.0-rc.5 Signed-off-by: Denis Kirillov --- app.go | 7 +++ downloader/download.go | 31 ++++++----- downloader/head.go | 8 +-- go.mod | 20 ++++--- go.sum | 41 +++++++++------ integration_test.go | 101 +++++++++++++++++++----------------- resolver/neofs.go | 17 ++---- tokens/bearer-token_test.go | 11 +++- uploader/upload.go | 39 +++++--------- utils/params.go | 2 + 10 files changed, 143 insertions(+), 134 deletions(-) diff --git a/app.go b/app.go index edb3d31..55ab986 100644 --- a/app.go +++ b/app.go @@ -18,6 +18,7 @@ import ( "github.com/nspcc-dev/neofs-http-gw/uploader" "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/pool" + "github.com/nspcc-dev/neofs-sdk-go/user" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -27,6 +28,7 @@ type ( app struct { log *zap.Logger pool *pool.Pool + owner *user.ID cfg *viper.Viper webServer *fasthttp.Server webDone chan struct{} @@ -102,6 +104,10 @@ func newApp(ctx context.Context, opt ...Option) App { a.log.Fatal("failed to get neofs credentials", zap.Error(err)) } + var owner user.ID + user.IDFromKey(&owner, key.PublicKey) + a.owner = &owner + var prm pool.InitParameters prm.SetKey(key) prm.SetNodeDialTimeout(a.cfg.GetDuration(cfgConTimeout)) @@ -319,6 +325,7 @@ func (a *app) AppParams() *utils.AppParams { return &utils.AppParams{ Logger: a.log, Pool: a.pool, + Owner: a.owner, Resolver: a.resolver, } } diff --git a/downloader/download.go b/downloader/download.go index 21726f4..f43d93d 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -24,7 +24,6 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/bearer" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" - "github.com/nspcc-dev/neofs-sdk-go/object/address" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/valyala/fasthttp" @@ -92,7 +91,7 @@ func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (str return http.DetectContentType(buf), buf, err // to not lose io.EOF } -func (r request) receiveFile(clnt *pool.Pool, objectAddress *address.Address) { +func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) { var ( err error dis = "inline" @@ -106,7 +105,7 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress *address.Address) { } var prm pool.PrmObjectGet - prm.SetAddress(*objectAddress) + prm.SetAddress(objectAddress) if btoken := bearerToken(r.RequestCtx); btoken != nil { prm.UseBearer(*btoken) } @@ -289,7 +288,7 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { // byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) { +func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Pool, oid.Address)) { var ( idCnr, _ = c.UserValue("cid").(string) idObj, _ = c.UserValue("oid").(string) @@ -310,9 +309,9 @@ func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Poo return } - addr := address.NewAddress() - addr.SetContainerID(*cnrID) - addr.SetObjectID(*objID) + var addr oid.Address + addr.SetContainer(*cnrID) + addr.SetObject(*objID) f(*d.newRequest(c, log), d.pool, addr) } @@ -323,7 +322,7 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { } // byAttribute is a wrapper similar to byAddress. -func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) { +func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.Pool, oid.Address)) { var ( scid, _ = c.UserValue("cid").(string) key, _ = url.QueryUnescape(c.UserValue("attr_key").(string)) @@ -362,11 +361,11 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.P return } - var addrObj address.Address - addrObj.SetContainerID(*containerID) - addrObj.SetObjectID(buf[0]) + var addrObj oid.Address + addrObj.SetContainer(*containerID) + addrObj.SetObject(buf[0]) - f(*d.newRequest(c, log), d.pool, &addrObj) + f(*d.newRequest(c, log), d.pool, addrObj) } func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) (*pool.ResObjectSearch, error) { @@ -433,12 +432,12 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { zipWriter := zip.NewWriter(w) var bufZip []byte - var addr address.Address + var addr oid.Address empty := true called := false btoken := bearerToken(c) - addr.SetContainerID(*containerID) + addr.SetContainer(*containerID) errIter := resSearch.Iterate(func(id oid.ID) bool { called = true @@ -448,7 +447,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { } empty = false - addr.SetObjectID(id) + addr.SetObject(id) if err = d.zipObject(zipWriter, addr, btoken, bufZip); err != nil { return true } @@ -477,7 +476,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { }) } -func (d *Downloader) zipObject(zipWriter *zip.Writer, addr address.Address, btoken *bearer.Token, bufZip []byte) error { +func (d *Downloader) zipObject(zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error { var prm pool.PrmObjectGet prm.SetAddress(addr) if btoken != nil { diff --git a/downloader/head.go b/downloader/head.go index d33d6d8..bcddd08 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -10,7 +10,7 @@ import ( "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/object" - "github.com/nspcc-dev/neofs-sdk-go/object/address" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -25,7 +25,7 @@ const ( hdrContainerID = "X-Container-Id" ) -func (r request) headObject(clnt *pool.Pool, objectAddress *address.Address) { +func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { var start = time.Now() if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) @@ -36,7 +36,7 @@ func (r request) headObject(clnt *pool.Pool, objectAddress *address.Address) { btoken := bearerToken(r.RequestCtx) var prm pool.PrmObjectHead - prm.SetAddress(*objectAddress) + prm.SetAddress(objectAddress) if btoken != nil { prm.UseBearer(*btoken) } @@ -77,7 +77,7 @@ func (r request) headObject(clnt *pool.Pool, objectAddress *address.Address) { if len(contentType) == 0 { contentType, _, err = readContentType(obj.PayloadSize(), func(sz uint64) (io.Reader, error) { var prmRange pool.PrmObjectRange - prmRange.SetAddress(*objectAddress) + prmRange.SetAddress(objectAddress) prmRange.SetLength(sz) if btoken != nil { prmRange.UseBearer(*btoken) diff --git a/go.mod b/go.mod index bf5bf37..89abe20 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.17 require ( github.com/fasthttp/router v1.4.1 - github.com/nspcc-dev/neo-go v0.98.0 - github.com/nspcc-dev/neofs-api-go/v2 v2.12.1 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220421125737-6e81e13e1bff + github.com/nspcc-dev/neo-go v0.99.1-pre.0.20220714084516-54849ef3e58e + github.com/nspcc-dev/neofs-api-go/v2 v2.13.0 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.5.0.20220725101411-90f1cc7a1a77 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.30.0 github.com/spf13/pflag v1.0.5 @@ -29,6 +29,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/containerd v1.6.2 // indirect + github.com/coreos/go-semver v0.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.8.1+incompatible // indirect @@ -39,13 +40,11 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/holiman/uint256 v1.2.0 // indirect github.com/klauspost/compress v1.15.0 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect @@ -55,13 +54,13 @@ require ( github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect - github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02 // indirect + github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect github.com/nspcc-dev/hrw v1.0.9 // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220713145417-4f184498bc42 // indirect + github.com/nspcc-dev/neofs-contract v0.15.3 // indirect github.com/nspcc-dev/neofs-crypto v0.3.0 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect - github.com/nspcc-dev/tzhash v1.5.2 // indirect - github.com/onsi/ginkgo v1.16.4 // indirect - github.com/onsi/gomega v1.15.0 // indirect + github.com/nspcc-dev/tzhash v1.6.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.1.1 // indirect @@ -81,7 +80,6 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/urfave/cli v1.22.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.7.0 // indirect @@ -97,5 +95,5 @@ require ( gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 0ee3fb9..ad66064 100644 --- a/go.sum +++ b/go.sum @@ -304,6 +304,7 @@ github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmeka github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -467,7 +468,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -587,7 +587,6 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -622,6 +621,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -732,30 +732,38 @@ github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= -github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02 h1:JgRx27vfGw5WV5QbaNDy0iy2WD1XJO964wwAapaYKLg= +github.com/nspcc-dev/dbft v0.0.0-20220629112714-fd49ca59d354/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= +github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 h1:n4ZaFCKt1pQJd7PXoMJabZWK9ejjbLOVrkl/lOUmshg= +github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= -github.com/nspcc-dev/neo-go v0.98.0 h1:yyW4sgY88/pLf0949qmgfkQXzRKC3CI/WyhqXNnwMd8= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= +github.com/nspcc-dev/neo-go v0.99.1-pre.0.20220714084516-54849ef3e58e h1:sjl8sniYkjoOsD8F+wzkpRfm9RzZSLQlh5Z/SMyf4A8= +github.com/nspcc-dev/neo-go v0.99.1-pre.0.20220714084516-54849ef3e58e/go.mod h1:/y5Sl8p3YheTygriBtCCMWKkDOek8HcvSo5ds2rJtKI= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220713145417-4f184498bc42 h1:Krhg2cD5mqvC+lin7irw6hj0M+x4ZOZrRGzrZQB+wcQ= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220713145417-4f184498bc42/go.mod h1:QBE0I30F2kOAISNpT5oks82yF4wkkUq3SCfI3Hqgx/Y= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.12.1 h1:PVU2rLlG9S0jDe5eKyaUs4nKo/la+mN5pvz32Gib3qM= -github.com/nspcc-dev/neofs-api-go/v2 v2.12.1/go.mod h1:73j09Xa7I2zQbM3HCvAHnDHPYiiWnEHa1d6Z6RDMBLU= +github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= +github.com/nspcc-dev/neofs-api-go/v2 v2.13.0 h1:7BcBiSjmWqJx0SPFlGRYt9ZFbMjucRIz9+Mv8UBLeq8= +github.com/nspcc-dev/neofs-api-go/v2 v2.13.0/go.mod h1:73j09Xa7I2zQbM3HCvAHnDHPYiiWnEHa1d6Z6RDMBLU= +github.com/nspcc-dev/neofs-contract v0.15.3 h1:7+NwyTtxFAnIevz0hR/XxQf6R2Ej2scjVR2bnnJnhBM= +github.com/nspcc-dev/neofs-contract v0.15.3/go.mod h1:BXVZUZUJxrmmDETglXHI8+5DSgn84B9y5DoSWqEjYCs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220421125737-6e81e13e1bff h1:rnkApn6vXUDcN9e/AsS/gaQZSe24VulI5IYrOkkvutM= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.3.0.20220421125737-6e81e13e1bff/go.mod h1:cQKdlr9Gmp5jxbOJ78S714i1AycfYUzpVddxVUD48WM= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.5.0.20220725101411-90f1cc7a1a77 h1:C8QL3pUN/QZ7OdXONEV2FQ+JamXoBbovvZiylWOCfBo= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.5.0.20220725101411-90f1cc7a1a77/go.mod h1:39SbCo+QUI0WJS47VsW4SCWhOwnJNpQSy8rGEG/b5vc= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= -github.com/nspcc-dev/tzhash v1.5.2 h1:GuIQPOY2xpl5ZE1pbUbz+QdKXVOTyzbbxSVv0nBfa98= -github.com/nspcc-dev/tzhash v1.5.2/go.mod h1:gwAx6mcsbkfY+JVp+PovoP2Gvw6y57W8dj7zDHKOhzI= +github.com/nspcc-dev/tzhash v1.6.1 h1:8dUrWFpjkmoHF+7GxuGUmarj9LLHWFcuyF3CTrqq9JE= +github.com/nspcc-dev/tzhash v1.6.1/go.mod h1:BoflzCVp+DO/f1mvbcsJQWoFzidIFBhWFZMglbUW648= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -771,7 +779,6 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -784,7 +791,6 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -1003,6 +1009,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= @@ -1126,6 +1133,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1183,6 +1191,7 @@ golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1317,6 +1326,7 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1417,6 +1427,7 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1573,7 +1584,6 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1587,8 +1597,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/gotestsum v1.7.0/go.mod h1:V1m4Jw3eBerhI/A6qCxUE07RnCg7ACkKj9BYcAm09V8= diff --git a/integration_test.go b/integration_test.go index 9031cc2..4e76163 100644 --- a/integration_test.go +++ b/integration_test.go @@ -10,18 +10,18 @@ import ( "mime/multipart" "net/http" "sort" - "strconv" "testing" "time" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neofs-sdk-go/container" + "github.com/nspcc-dev/neofs-sdk-go/container/acl" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" - "github.com/nspcc-dev/neofs-sdk-go/object/address" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/nspcc-dev/neofs-sdk-go/policy" "github.com/nspcc-dev/neofs-sdk-go/pool" + "github.com/nspcc-dev/neofs-sdk-go/user" "github.com/spf13/viper" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" @@ -44,28 +44,30 @@ func TestIntegration(t *testing.T) { rootCtx := context.Background() aioImage := "nspccdev/neofs-aio-testcontainer:" versions := []string{ - "0.24.0", - "0.25.1", - "0.26.1", "0.27.5", + "0.28.1", + "0.29.0", "latest", } key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) + var ownerID user.ID + user.IDFromKey(&ownerID, key.PrivateKey.PublicKey) + for _, version := range versions { ctx, cancel2 := context.WithCancel(rootCtx) aioContainer := createDockerContainer(ctx, t, aioImage+version) cancel := runServer() clientPool := getPool(ctx, t, key) - CID, err := createContainer(ctx, t, clientPool, version) + CID, err := createContainer(ctx, t, clientPool, ownerID, version) require.NoError(t, err, version) t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID, version) }) - t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID, version) }) - t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID, version) }) - t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID, version) }) + t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, ownerID, CID, version) }) + t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID, version) }) + t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, ownerID, CID, version) }) cancel() err = aioContainer.Terminate(ctx) @@ -144,14 +146,14 @@ func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnr err = id.DecodeString(addr.OID) require.NoError(t, err) - objectAddress := address.NewAddress() - objectAddress.SetContainerID(*cnrID) - objectAddress.SetObjectID(*id) + var objectAddress oid.Address + objectAddress.SetContainer(*cnrID) + objectAddress.SetObject(*id) payload := bytes.NewBuffer(nil) var prm pool.PrmObjectGet - prm.SetAddress(*objectAddress) + prm.SetAddress(objectAddress) res, err := p.GetObject(ctx, prm) require.NoError(t, err) @@ -166,13 +168,13 @@ func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnr } } -func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) { +func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID *cid.ID, version string) { content := "content of file" attributes := map[string]string{ "some-attr": "some-get-value", } - id := putObject(ctx, t, clientPool, CID, content, attributes) + id := putObject(ctx, t, clientPool, ownerID, CID, content, attributes) resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + id.String()) require.NoError(t, err) @@ -200,12 +202,27 @@ func checkGetResponse(t *testing.T, resp *http.Response, content string, attribu } } -func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) { +func checkGetByAttrResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) { + defer func() { + err := resp.Body.Close() + require.NoError(t, err) + }() + + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, content, string(data)) + + for k, v := range attributes { + require.Equal(t, v, resp.Header.Get(k)) + } +} + +func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID *cid.ID, version string) { keyAttr, valAttr := "some-attr", "some-get-by-attr-value" content := "content of file" attributes := map[string]string{keyAttr: valAttr} - id := putObject(ctx, t, clientPool, CID, content, attributes) + id := putObject(ctx, t, clientPool, ownerID, CID, content, attributes) expectedAttr := map[string]string{ "X-Attribute-" + keyAttr: valAttr, @@ -224,29 +241,14 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci } } -func checkGetByAttrResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) { - defer func() { - err := resp.Body.Close() - require.NoError(t, err) - }() - - data, err := io.ReadAll(resp.Body) - require.NoError(t, err) - require.Equal(t, content, string(data)) - - for k, v := range attributes { - require.Equal(t, v, resp.Header.Get(k)) - } -} - -func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) { +func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID *cid.ID, version string) { names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"} contents := []string{"content of file1", "content of file2"} attributes1 := map[string]string{attributeFilePath: names[0]} attributes2 := map[string]string{attributeFilePath: names[1]} - putObject(ctx, t, clientPool, CID, contents[0], attributes1) - putObject(ctx, t, clientPool, CID, contents[1], attributes2) + putObject(ctx, t, clientPool, ownerID, CID, contents[0], attributes1) + putObject(ctx, t, clientPool, ownerID, CID, contents[1], attributes2) baseURL := "http://localhost:8082/zip/" + CID.String() makeZipTest(t, baseURL, names, contents) @@ -348,18 +350,23 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool return clientPool } -func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, version string) (*cid.ID, error) { - pp, err := policy.Parse("REP 1") +func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, version string) (*cid.ID, error) { + var policy netmap.PlacementPolicy + err := policy.DecodeString("REP 1") require.NoError(t, err) - cnr := container.New( - container.WithPolicy(pp), - container.WithCustomBasicACL(0x0FFFFFFF), - container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10))) - cnr.SetOwnerID(clientPool.OwnerID()) + var cnr container.Container + cnr.Init() + cnr.SetPlacementPolicy(policy) + cnr.SetBasicACL(acl.PublicRWExtended) + cnr.SetOwner(ownerID) + + container.SetCreationTime(&cnr, time.Now()) if version >= versionWithNativeNames { - container.SetNativeName(cnr, testContainerName) + var domain container.Domain + domain.SetName(testContainerName) + container.WriteDomain(&cnr, domain) } var waitPrm pool.WaitParams @@ -367,7 +374,7 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, v waitPrm.SetPollInterval(3 * time.Second) var prm pool.PrmContainerPut - prm.SetContainer(*cnr) + prm.SetContainer(cnr) prm.SetWaitParams(waitPrm) CID, err := clientPool.PutContainer(ctx, prm) @@ -379,10 +386,10 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, v return CID, err } -func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, content string, attributes map[string]string) *oid.ID { +func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID *cid.ID, content string, attributes map[string]string) *oid.ID { obj := object.New() obj.SetContainerID(*CID) - obj.SetOwnerID(clientPool.OwnerID()) + obj.SetOwnerID(&ownerID) var attrs []object.Attribute for key, val := range attributes { diff --git a/resolver/neofs.go b/resolver/neofs.go index caa460b..6931baa 100644 --- a/resolver/neofs.go +++ b/resolver/neofs.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" - "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/pool" ) @@ -27,20 +26,10 @@ func (x *NeoFSResolver) SystemDNS(ctx context.Context) (string, error) { return "", fmt.Errorf("read network info via client: %w", err) } - var domain string - - networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool { - if string(parameter.Key()) == "SystemDNS" { - domain = string(parameter.Value()) - return true - } - - return false - }) - - if domain == "" { + domain := networkInfo.RawNetworkParameter("SystemDNS") + if domain == nil { return "", errors.New("system DNS parameter not found or empty") } - return domain, nil + return string(domain), nil } diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index 5ecf1dd..59aabad 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "testing" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neofs-sdk-go/bearer" "github.com/nspcc-dev/neofs-sdk-go/user" "github.com/stretchr/testify/require" @@ -59,10 +60,13 @@ func Test_fromHeader(t *testing.T) { } func Test_fetchBearerToken(t *testing.T) { + key, err := keys.NewPrivateKey() + require.NoError(t, err) var uid user.ID + user.IDFromKey(&uid, key.PrivateKey.PublicKey) tkn := new(bearer.Token) - tkn.SetOwnerID(uid) + tkn.ForUser(uid) t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) require.NotEmpty(t, t64) @@ -133,10 +137,13 @@ func makeTestRequest(cookie, header string) *fasthttp.RequestCtx { } func Test_checkAndPropagateBearerToken(t *testing.T) { + key, err := keys.NewPrivateKey() + require.NoError(t, err) var uid user.ID + user.IDFromKey(&uid, key.PrivateKey.PublicKey) tkn := new(bearer.Token) - tkn.SetOwnerID(uid) + tkn.ForUser(uid) t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) require.NotEmpty(t, t64) diff --git a/uploader/upload.go b/uploader/upload.go index d60b62f..d6b06c8 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -2,7 +2,6 @@ package uploader import ( "context" - "encoding/binary" "encoding/json" "fmt" "io" @@ -14,9 +13,7 @@ import ( "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/bearer" - "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" - "github.com/nspcc-dev/neofs-sdk-go/object/address" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/nspcc-dev/neofs-sdk-go/user" @@ -34,6 +31,7 @@ type Uploader struct { appCtx context.Context log *zap.Logger pool *pool.Pool + ownerID *user.ID enableDefaultTimestamp bool containerResolver *resolver.ContainerResolver } @@ -51,6 +49,7 @@ func New(ctx context.Context, params *utils.AppParams, enableDefaultTimestamp bo appCtx: ctx, log: params.Logger, pool: params.Pool, + ownerID: params.Owner, enableDefaultTimestamp: enableDefaultTimestamp, containerResolver: params.Resolver, } @@ -61,7 +60,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { var ( file MultipartFile idObj *oid.ID - addr = address.NewAddress() + addr oid.Address scid, _ = c.UserValue("cid").(string) log = u.log.With(zap.String("cid", scid)) bodyStream = c.RequestBodyStream() @@ -158,8 +157,8 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { return } - addr.SetObjectID(*idObj) - addr.SetContainerID(*idCnr) + addr.SetObject(*idObj) + addr.SetContainer(*idCnr) // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { @@ -187,10 +186,10 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*user.ID, *bearer.Token) { if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil { - issuer, _ := tkn.Issuer() + issuer := bearer.ResolveIssuer(*tkn) return &issuer, tkn } - return u.pool.OwnerID(), nil + return u.ownerID, nil } type putResponse struct { @@ -198,12 +197,10 @@ type putResponse struct { ContainerID string `json:"container_id"` } -func newPutResponse(addr *address.Address) *putResponse { - objID, _ := addr.ObjectID() - cnrID, _ := addr.ContainerID() +func newPutResponse(addr oid.Address) *putResponse { return &putResponse{ - ObjectID: objID.String(), - ContainerID: cnrID.String(), + ObjectID: addr.Object().EncodeToString(), + ContainerID: addr.Container().EncodeToString(), } } @@ -220,21 +217,13 @@ func getEpochDurations(ctx context.Context, p *pool.Pool) (*epochDurations, erro } res := &epochDurations{ - currentEpoch: networkInfo.CurrentEpoch(), - msPerBlock: networkInfo.MsPerBlock(), + currentEpoch: networkInfo.CurrentEpoch(), + msPerBlock: networkInfo.MsPerBlock(), + blockPerEpoch: networkInfo.EpochDuration(), } - networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool { - if string(parameter.Key()) == "EpochDuration" { - data := make([]byte, 8) - copy(data, parameter.Value()) - res.blockPerEpoch = binary.LittleEndian.Uint64(data) - return true - } - return false - }) if res.blockPerEpoch == 0 { - return nil, fmt.Errorf("not found param: EpochDuration") + return nil, fmt.Errorf("EpochDuration is empty") } return res, nil } diff --git a/utils/params.go b/utils/params.go index 4bcd352..7c6c6eb 100644 --- a/utils/params.go +++ b/utils/params.go @@ -3,11 +3,13 @@ package utils import ( "github.com/nspcc-dev/neofs-http-gw/resolver" "github.com/nspcc-dev/neofs-sdk-go/pool" + "github.com/nspcc-dev/neofs-sdk-go/user" "go.uber.org/zap" ) type AppParams struct { Logger *zap.Logger Pool *pool.Pool + Owner *user.ID Resolver *resolver.ContainerResolver } From 98991d63ae7ede4f51d0c2c51310e0cc37791356 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 25 Jul 2022 17:04:09 +0300 Subject: [PATCH 286/548] Release v0.22.0 Signed-off-by: Alex Vanin --- CHANGELOG.md | 11 +++++++++++ VERSION | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cb81b9..d39f66c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ This document outlines major changes between releases. +## [0.22.0] - 2022-07-25 + +### Added +- Default params documentation (#172) +- Health metric (#175) + +### Changed +- Version output (#169) +- Updated SDK Version (#178) + ## [0.21.0] - 2022-06-20 ### Fixed @@ -179,3 +189,4 @@ releases. [0.19.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.18.0...v0.19.0 [0.20.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.19.0...v0.20.0 [0.21.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.20.0...v0.21.0 +[0.22.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.21.0...v0.22.0 diff --git a/VERSION b/VERSION index 759e855..4f27943 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.21.0 +v0.22.0 From f0e8bde7619b97427068dea75348c2f34a2b96a2 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 27 Jul 2022 18:54:44 +0300 Subject: [PATCH 287/548] [#182] Drop GO111MODULE from builds It is enabled by default since go1.16 Signed-off-by: Alex Vanin --- Makefile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Makefile b/Makefile index a5489d1..c9d9be4 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,6 @@ all: $(BINS) $(BINS): $(DIRS) dep @echo "⇒ Build $@" CGO_ENABLED=0 \ - GO111MODULE=on \ go build -v -trimpath \ -ldflags "-X main.Version=$(VERSION)" \ -o $@ ./ @@ -35,11 +34,9 @@ $(DIRS): dep: @printf "⇒ Download requirements: " @CGO_ENABLED=0 \ - GO111MODULE=on \ go mod download && echo OK @printf "⇒ Tidy requirements: " @CGO_ENABLED=0 \ - GO111MODULE=on \ go mod tidy -v && echo OK docker/%: @@ -65,7 +62,7 @@ cover: # Reformat code fmt: @echo "⇒ Processing gofmt check" - @GO111MODULE=on gofmt -s -w ./ + @gofmt -s -w ./ # Build clean Docker image image: From af732d294c5c512ae9e41083dae19f6e8dfe3152 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 27 Jul 2022 09:52:08 +0300 Subject: [PATCH 288/548] [#171] Sync metrics and pprof configuration Signed-off-by: Denis Kirillov --- app.go | 65 +++++++++++-------- config/config.env | 9 ++- config/config.yaml | 9 ++- docs/gate-configuration.md | 38 +++++++++-- metrics.go | 127 ------------------------------------- metrics/metrics.go | 68 ++++++++++++++++++++ metrics/pprof.go | 33 ++++++++++ metrics/service.go | 44 +++++++++++++ pprof.go | 44 ------------- settings.go | 20 ++++++ 10 files changed, 248 insertions(+), 209 deletions(-) delete mode 100644 metrics.go create mode 100644 metrics/metrics.go create mode 100644 metrics/pprof.go create mode 100644 metrics/service.go delete mode 100644 pprof.go diff --git a/app.go b/app.go index 55ab986..ce51cef 100644 --- a/app.go +++ b/app.go @@ -13,6 +13,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/nspcc-dev/neofs-http-gw/downloader" + "github.com/nspcc-dev/neofs-http-gw/metrics" "github.com/nspcc-dev/neofs-http-gw/resolver" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/uploader" @@ -162,8 +163,8 @@ func newApp(ctx context.Context, opt ...Option) App { a.log.Info("container resolver is disabled") } - if a.cfg.GetBool(cmdMetrics) { - a.metrics = newGateMetrics() + if a.cfg.GetBool(cfgPrometheusEnabled) { + a.metrics = metrics.NewGateMetrics() } return a @@ -254,11 +255,6 @@ func (a *app) Wait() { } func (a *app) Serve(ctx context.Context) { - go func() { - <-ctx.Done() - a.log.Info("shutting down web server", zap.Error(a.webServer.Shutdown())) - close(a.webDone) - }() edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) uploadRoutes := uploader.New(ctx, a.AppParams(), edts) downloadSettings := downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)} @@ -282,32 +278,45 @@ func (a *app) Serve(ctx context.Context) { a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") r.GET("/zip/{cid}/{prefix:*}", a.logger(downloadRoutes.DownloadZipped)) a.log.Info("added path /zip/{cid}/{prefix}") - // enable metrics - if a.cfg.GetBool(cmdMetrics) { - a.log.Info("added path /metrics/") - attachMetrics(r, a.log) - } - // enable pprof - if a.cfg.GetBool(cmdPprof) { - a.log.Info("added path /debug/pprof/") - attachProfiler(r) - } + + pprofConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPprofEnabled), Address: a.cfg.GetString(cfgPprofAddress)} + pprof := metrics.NewPprofService(a.log, pprofConfig) + prometheusConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPrometheusEnabled), Address: a.cfg.GetString(cfgPrometheusAddress)} + prometheus := metrics.NewPrometheusService(a.log, prometheusConfig) + bind := a.cfg.GetString(cfgListenAddress) tlsCertPath := a.cfg.GetString(cfgTLSCertificate) tlsKeyPath := a.cfg.GetString(cfgTLSKey) a.webServer.Handler = r.Handler - var err error - if tlsCertPath == "" && tlsKeyPath == "" { - a.log.Info("running web server", zap.String("address", bind)) - err = a.webServer.ListenAndServe(bind) - } else { - a.log.Info("running web server (TLS-enabled)", zap.String("address", bind)) - err = a.webServer.ListenAndServeTLS(bind, tlsCertPath, tlsKeyPath) - } - if err != nil { - a.log.Fatal("could not start server", zap.Error(err)) - } + + go pprof.Start() + go prometheus.Start() + + go func() { + var err error + if tlsCertPath == "" && tlsKeyPath == "" { + a.log.Info("running web server", zap.String("address", bind)) + err = a.webServer.ListenAndServe(bind) + } else { + a.log.Info("running web server (TLS-enabled)", zap.String("address", bind)) + err = a.webServer.ListenAndServeTLS(bind, tlsCertPath, tlsKeyPath) + } + if err != nil { + a.log.Fatal("could not start server", zap.Error(err)) + } + }() + + <-ctx.Done() + a.log.Info("shutting down web server", zap.Error(a.webServer.Shutdown())) + + ctx, cancel := context.WithTimeout(context.Background(), defaultShutdownTimeout) + defer cancel() + + pprof.ShutDown(ctx) + prometheus.ShutDown(ctx) + + close(a.webDone) } func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { diff --git a/config/config.env b/config/config.env index f4f0f07..7f7b406 100644 --- a/config/config.env +++ b/config/config.env @@ -8,9 +8,12 @@ HTTP_GW_WALLET_ADDRESS=NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP HTTP_GW_WALLET_PASSPHRASE=pwd # Enable metrics. -HTTP_GW_METRICS=true -# Enable pprof. -HTTP_GW_PPROF=true +HTTP_GW_PPROF_ENABLED=true +HTTP_GW_PPROF_ADDRESS=localhost:8083 + +HTTP_GW_PROMETHEUS_ENABLED=true +HTTP_GW_PROMETHEUS_ADDRESS=localhost:8084 + # Log level. HTTP_GW_LOGGER_LEVEL=debug diff --git a/config/config.yaml b/config/config.yaml index ef77302..d3b5e9a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -3,8 +3,13 @@ wallet: address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP # Account address. If omitted default one will be used. passphrase: pwd # Passphrase to decrypt wallet. If you're using a wallet without a password, place '' here. -metrics: true # Enable metrics. -pprof: true # Enable pprof. +pprof: + enabled: true # Enable pprof. + address: localhost:8083 +prometheus: + enabled: true # Enable metrics. + address: localhost:8084 + logger: level: debug # Log level. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 0950e5d..040ae1b 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -19,6 +19,9 @@ There are some custom types used for brevity: | `web` | [Web configuration](#web-section) | | `upload-header` | [Upload header configuration](#upload-header-section) | | `zip` | [ZIP configuration](#zip-section) | +| `pprof` | [Pprof configuration](#pprof-section) | +| `prometheus` | [Prometheus configuration](#prometheus-section) | + # General section @@ -32,9 +35,6 @@ resolve_order: - nns - dns -metrics: false -pprof: false - connect_timeout: 5s request_timeout: 5s rebalance_timer: 30s @@ -47,8 +47,6 @@ rebalance_timer: 30s | `tls_key` | `string` | | Path to the TLS key. | | `rpc_endpoint` | `string` | | The address of the RPC host to which the gateway connects to resolve bucket names. | | `resolve_order` | `[]string` | `[nns, dns]` | Order of bucket name resolvers to use. | -| `metrics` | `bool` | `false` | Flag to enable and expose the prometheus metrics. | -| `pprof` | `bool` | `false` | Flag to enable the profiler. | | `connect_timeout` | `duration` | `10s` | Timeout to connect to a node. | | `request_timeout` | `duration` | `15s` | Timeout to check node health during rebalance. | | `rebalance_timer` | `duration` | `60s` | Interval to check node health. | @@ -156,3 +154,33 @@ zip: |---------------|--------|---------------|--------------------------------------------------------------| | `compression` | `bool` | `false` | Enable zip compression when download files by common prefix. | + +# `pprof` section + +Contains configuration for the `pprof` profiler. + +```yaml +pprof: + enabled: true + address: localhost:8083 +``` + +| Parameter | Type | Default value | Description | +|-----------|----------|------------------|-----------------------------------------| +| `enabled` | `bool` | `false` | Flag to enable the service. | +| `address` | `string` | `localhost:8083` | Address that service listener binds to. | + +# `prometheus` section + +Contains configuration for the `prometheus` metrics service. + +```yaml +prometheus: + enabled: true + address: localhost:8084 +``` + +| Parameter | Type | Default value | Description | +|-----------|----------|------------------|-----------------------------------------| +| `enabled` | `bool` | `false` | Flag to enable the service. | +| `address` | `string` | `localhost:8084` | Address that service listener binds to. | diff --git a/metrics.go b/metrics.go deleted file mode 100644 index bb92b6a..0000000 --- a/metrics.go +++ /dev/null @@ -1,127 +0,0 @@ -package main - -import ( - "github.com/fasthttp/router" - "github.com/nspcc-dev/neofs-http-gw/response" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/expfmt" - "github.com/valyala/fasthttp" - "go.uber.org/zap" -) - -const ( - namespace = "neofs_http_gw" - stateSubsystem = "state" -) - -type GateMetrics struct { - stateMetrics -} - -type stateMetrics struct { - healthCheck prometheus.Gauge -} - -func newGateMetrics() *GateMetrics { - stateMetric := newStateMetrics() - stateMetric.register() - - return &GateMetrics{ - stateMetrics: *stateMetric, - } -} - -func newStateMetrics() *stateMetrics { - return &stateMetrics{ - healthCheck: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: stateSubsystem, - Name: "health", - Help: "Current HTTP gateway state", - }), - } -} - -func (m stateMetrics) register() { - prometheus.MustRegister(m.healthCheck) -} - -func (m stateMetrics) SetHealth(s int32) { - m.healthCheck.Set(float64(s)) -} - -func attachMetrics(r *router.Router, l *zap.Logger) { - r.GET("/metrics/", metricsHandler(prometheus.DefaultGatherer, l)) -} - -func metricsHandler(reg prometheus.Gatherer, logger *zap.Logger) fasthttp.RequestHandler { - var ( - inFlightSem chan struct{} - errCnt = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "promhttp_metric_handler_errors_total", - Help: "Total number of internal errors encountered by the promhttp metric handler.", - }, - []string{"cause"}, - ) - ) - - h := fasthttp.RequestHandler(func(c *fasthttp.RequestCtx) { - if inFlightSem != nil { - select { - case inFlightSem <- struct{}{}: // All good, carry on. - defer func() { <-inFlightSem }() - default: - response.Error(c, "Limit of concurrent requests reached, try again later.", fasthttp.StatusServiceUnavailable) - return - } - } - mfs, err := reg.Gather() - if err != nil { - if logger != nil { - panic("error gathering metrics:" + err.Error()) - } - - errCnt.WithLabelValues("gathering").Inc() - response.Error(c, err.Error(), fasthttp.StatusServiceUnavailable) - return - } - - contentType := expfmt.FmtText - c.SetContentType(string(contentType)) - enc := expfmt.NewEncoder(c, contentType) - - var lastErr error - - // handleError handles the error according to opts.ErrorHandling - // and returns true if we have to abort after the handling. - handleError := func(err error) bool { - if err == nil { - return false - } - lastErr = err - if logger != nil { - logger.Error("encoding and sending metric family", zap.Error(err)) - } - errCnt.WithLabelValues("encoding").Inc() - response.Error(c, err.Error(), fasthttp.StatusServiceUnavailable) - return true - } - - for _, mf := range mfs { - if handleError(enc.Encode(mf)) { - return - } - } - if closer, ok := enc.(expfmt.Closer); ok { - // This in particular takes care of the final "# EOF\n" line for OpenMetrics. - if handleError(closer.Close()) { - return - } - } - - handleError(lastErr) - }) - - return h -} diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 0000000..480cd81 --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,68 @@ +package metrics + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.uber.org/zap" +) + +const ( + namespace = "neofs_http_gw" + stateSubsystem = "state" +) + +type GateMetrics struct { + stateMetrics +} + +type stateMetrics struct { + healthCheck prometheus.Gauge +} + +// NewGateMetrics creates new metrics for http gate. +func NewGateMetrics() *GateMetrics { + stateMetric := newStateMetrics() + stateMetric.register() + + return &GateMetrics{ + stateMetrics: *stateMetric, + } +} + +func newStateMetrics() *stateMetrics { + return &stateMetrics{ + healthCheck: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: stateSubsystem, + Name: "health", + Help: "Current HTTP gateway state", + }), + } +} + +func (m stateMetrics) register() { + prometheus.MustRegister(m.healthCheck) +} + +func (m stateMetrics) SetHealth(s int32) { + m.healthCheck.Set(float64(s)) +} + +// NewPrometheusService creates a new service for gathering prometheus metrics. +func NewPrometheusService(log *zap.Logger, cfg Config) *Service { + if log == nil { + return nil + } + + return &Service{ + Server: &http.Server{ + Addr: cfg.Address, + Handler: promhttp.Handler(), + }, + enabled: cfg.Enabled, + serviceType: "Prometheus", + log: log.With(zap.String("service", "Prometheus")), + } +} diff --git a/metrics/pprof.go b/metrics/pprof.go new file mode 100644 index 0000000..4719a69 --- /dev/null +++ b/metrics/pprof.go @@ -0,0 +1,33 @@ +package metrics + +import ( + "net/http" + "net/http/pprof" + + "go.uber.org/zap" +) + +// NewPprofService creates a new service for gathering pprof metrics. +func NewPprofService(l *zap.Logger, cfg Config) *Service { + handler := http.NewServeMux() + handler.HandleFunc("/debug/pprof/", pprof.Index) + handler.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + handler.HandleFunc("/debug/pprof/profile", pprof.Profile) + handler.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + handler.HandleFunc("/debug/pprof/trace", pprof.Trace) + + // Manually add support for paths linked to by index page at /debug/pprof/ + for _, item := range []string{"allocs", "block", "heap", "goroutine", "mutex", "threadcreate"} { + handler.Handle("/debug/pprof/"+item, pprof.Handler(item)) + } + + return &Service{ + Server: &http.Server{ + Addr: cfg.Address, + Handler: handler, + }, + enabled: cfg.Enabled, + serviceType: "Pprof", + log: l.With(zap.String("service", "Pprof")), + } +} diff --git a/metrics/service.go b/metrics/service.go new file mode 100644 index 0000000..7cb3ca2 --- /dev/null +++ b/metrics/service.go @@ -0,0 +1,44 @@ +package metrics + +import ( + "context" + "net/http" + + "go.uber.org/zap" +) + +// Service serves metrics. +type Service struct { + *http.Server + enabled bool + log *zap.Logger + serviceType string +} + +// Config is a params to configure service. +type Config struct { + Address string + Enabled bool +} + +// Start runs http service with the exposed endpoint on the configured port. +func (ms *Service) Start() { + if ms.enabled { + ms.log.Info("service is running", zap.String("endpoint", ms.Addr)) + err := ms.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + ms.log.Warn("service couldn't start on configured port") + } + } else { + ms.log.Info("service hasn't started since it's disabled") + } +} + +// ShutDown stops the service. +func (ms *Service) ShutDown(ctx context.Context) { + ms.log.Info("shutting down service", zap.String("endpoint", ms.Addr)) + err := ms.Shutdown(ctx) + if err != nil { + ms.log.Panic("can't shut down service") + } +} diff --git a/pprof.go b/pprof.go deleted file mode 100644 index 8831314..0000000 --- a/pprof.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "net/http/pprof" - rtp "runtime/pprof" - - "github.com/fasthttp/router" - "github.com/nspcc-dev/neofs-http-gw/response" - "github.com/valyala/fasthttp" - "github.com/valyala/fasthttp/fasthttpadaptor" -) - -func attachProfiler(r *router.Router) { - r.GET("/debug/pprof/", pprofHandler()) - r.GET("/debug/pprof/{name}/", pprofHandler()) -} - -func pprofHandler() fasthttp.RequestHandler { - items := rtp.Profiles() - - profiles := map[string]fasthttp.RequestHandler{ - "": fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Index), - "cmdline": fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Cmdline), - "profile": fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Profile), - "symbol": fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Symbol), - "trace": fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Trace), - } - - for i := range items { - name := items[i].Name() - profiles[name] = fasthttpadaptor.NewFastHTTPHandler(pprof.Handler(name)) - } - - return func(ctx *fasthttp.RequestCtx) { - name, _ := ctx.UserValue("name").(string) - - if handler, ok := profiles[name]; ok { - handler(ctx) - return - } - - response.Error(ctx, "Not found", fasthttp.StatusNotFound) - } -} diff --git a/settings.go b/settings.go index 1b0c8b9..e99f701 100644 --- a/settings.go +++ b/settings.go @@ -20,6 +20,8 @@ const ( defaultRequestTimeout = 15 * time.Second defaultConnectTimeout = 10 * time.Second + defaultShutdownTimeout = 15 * time.Second + cfgListenAddress = "listen_address" cfgTLSCertificate = "tls_certificate" cfgTLSKey = "tls_key" @@ -32,6 +34,12 @@ const ( cfgWebStreamRequestBody = "web.stream_request_body" cfgWebMaxRequestBodySize = "web.max_request_body_size" + // Metrics / Profiler. + cfgPrometheusEnabled = "prometheus.enabled" + cfgPrometheusAddress = "prometheus.address" + cfgPprofEnabled = "pprof.enabled" + cfgPprofAddress = "pprof.address" + // Timeouts. cfgConTimeout = "connect_timeout" cfgReqTimeout = "request_timeout" @@ -128,6 +136,18 @@ func settings() *viper.Viper { // zip: v.SetDefault(cfgZipCompression, false) + // metrics + v.SetDefault(cfgPprofAddress, "localhost:8083") + v.SetDefault(cfgPrometheusAddress, "localhost:8084") + + // Binding flags + if err := v.BindPFlag(cfgPprofEnabled, flags.Lookup(cmdPprof)); err != nil { + panic(err) + } + if err := v.BindPFlag(cfgPrometheusEnabled, flags.Lookup(cmdMetrics)); err != nil { + panic(err) + } + if err := v.BindPFlags(flags); err != nil { panic(err) } From 0f7737088d23db1a4463b18342d8d1e89b399f6c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 29 Jul 2022 09:34:26 +0300 Subject: [PATCH 289/548] [#184] Add config param for pool error threshold Signed-off-by: Denis Kirillov --- app.go | 1 + config/config.env | 2 ++ config/config.yaml | 1 + docs/gate-configuration.md | 24 +++++++++++++----------- settings.go | 14 ++++++++++---- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/app.go b/app.go index ce51cef..bd9dd1d 100644 --- a/app.go +++ b/app.go @@ -114,6 +114,7 @@ func newApp(ctx context.Context, opt ...Option) App { prm.SetNodeDialTimeout(a.cfg.GetDuration(cfgConTimeout)) prm.SetHealthcheckTimeout(a.cfg.GetDuration(cfgReqTimeout)) prm.SetClientRebalanceInterval(a.cfg.GetDuration(cfgRebalance)) + prm.SetErrorThreshold(a.cfg.GetUint32(cfgPoolErrorThreshold)) for i := 0; ; i++ { address := a.cfg.GetString(cfgPeers + "." + strconv.Itoa(i) + ".address") diff --git a/config/config.env b/config/config.env index 7f7b406..d6258fb 100644 --- a/config/config.env +++ b/config/config.env @@ -83,6 +83,8 @@ HTTP_GW_CONNECT_TIMEOUT=5s HTTP_GW_REQUEST_TIMEOUT=5s # Interval to check nodes health. HTTP_GW_REBALANCE_TIMER=30s +# The number of errors on connection after which node is considered as unhealthy +S3_GW_POOL_ERROR_THRESHOLD=100 # Enable zip compression to download files by common prefix. HTTP_GW_ZIP_COMPRESSION=false diff --git a/config/config.yaml b/config/config.yaml index d3b5e9a..ef784ed 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -84,6 +84,7 @@ upload_header: connect_timeout: 5s # Timeout to dial node. request_timeout: 5s # Timeout to check node health during rebalance. rebalance_timer: 30s # Interval to check nodes health. +pool_error_threshold: 100 # The number of errors on connection after which node is considered as unhealthy. zip: compression: false # Enable zip compression to download files by common prefix. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 040ae1b..4661aa5 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -37,19 +37,21 @@ resolve_order: connect_timeout: 5s request_timeout: 5s -rebalance_timer: 30s +rebalance_timer: 30s +pool_error_threshold: 100 ``` -| Parameter | Type | Default value | Description | -|-------------------|------------|----------------|------------------------------------------------------------------------------------| -| `listen_address` | `string` | `0.0.0.0:8082` | The address that the gateway is listening on. | -| `tls_certificate` | `string` | | Path to the TLS certificate. | -| `tls_key` | `string` | | Path to the TLS key. | -| `rpc_endpoint` | `string` | | The address of the RPC host to which the gateway connects to resolve bucket names. | -| `resolve_order` | `[]string` | `[nns, dns]` | Order of bucket name resolvers to use. | -| `connect_timeout` | `duration` | `10s` | Timeout to connect to a node. | -| `request_timeout` | `duration` | `15s` | Timeout to check node health during rebalance. | -| `rebalance_timer` | `duration` | `60s` | Interval to check node health. | +| Parameter | Type | Default value | Description | +|------------------------|------------|----------------|------------------------------------------------------------------------------------| +| `listen_address` | `string` | `0.0.0.0:8082` | The address that the gateway is listening on. | +| `tls_certificate` | `string` | | Path to the TLS certificate. | +| `tls_key` | `string` | | Path to the TLS key. | +| `rpc_endpoint` | `string` | | The address of the RPC host to which the gateway connects to resolve bucket names. | +| `resolve_order` | `[]string` | `[nns, dns]` | Order of bucket name resolvers to use. | +| `connect_timeout` | `duration` | `10s` | Timeout to connect to a node. | +| `request_timeout` | `duration` | `15s` | Timeout to check node health during rebalance. | +| `rebalance_timer` | `duration` | `60s` | Interval to check node health. | +| `pool_error_threshold` | `uint32` | `100` | The number of errors on connection after which node is considered as unhealthy. | # `wallet` section diff --git a/settings.go b/settings.go index e99f701..c6ad084 100644 --- a/settings.go +++ b/settings.go @@ -22,6 +22,8 @@ const ( defaultShutdownTimeout = 15 * time.Second + defaultPoolErrorThreshold uint32 = 100 + cfgListenAddress = "listen_address" cfgTLSCertificate = "tls_certificate" cfgTLSKey = "tls_key" @@ -40,10 +42,11 @@ const ( cfgPprofEnabled = "pprof.enabled" cfgPprofAddress = "pprof.address" - // Timeouts. - cfgConTimeout = "connect_timeout" - cfgReqTimeout = "request_timeout" - cfgRebalance = "rebalance_timer" + // Pool config. + cfgConTimeout = "connect_timeout" + cfgReqTimeout = "request_timeout" + cfgRebalance = "rebalance_timer" + cfgPoolErrorThreshold = "pool_error_threshold" // Logger. cfgLoggerLevel = "logger.level" @@ -122,6 +125,9 @@ func settings() *viper.Viper { // logger: v.SetDefault(cfgLoggerLevel, "debug") + // pool: + v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) + // web-server: v.SetDefault(cfgWebReadBufferSize, 4096) v.SetDefault(cfgWebWriteBufferSize, 4096) From 21b0af9d41e312afe540223bdd9020d6960c9b4e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 2 Aug 2022 10:38:50 +0300 Subject: [PATCH 290/548] Release v0.23.0 Signed-off-by: Denis Kirillov --- CHANGELOG.md | 15 +++++++++++++++ VERSION | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d39f66c..1fd90fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ This document outlines major changes between releases. +## [0.23.0] - 2022-08-02 + +### Added +- New param to configure pool error threshold (#184) + +### Changed +- Pprof and prometheus metrics configuration (#171) +- Drop GO111MODULES from builds (#182) + +### Updating from v0.22.0 +1. To enable pprof use `pprof.enabled` instead of `pprof` in config. + To enable prometheus metrics use `prometheus.enabled` instead of `metrics` in config. + If you are using the command line flags you can skip this step. + ## [0.22.0] - 2022-07-25 ### Added @@ -190,3 +204,4 @@ releases. [0.20.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.19.0...v0.20.0 [0.21.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.20.0...v0.21.0 [0.22.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.21.0...v0.22.0 +[0.23.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.22.0...v0.23.0 diff --git a/VERSION b/VERSION index 4f27943..0c2a959 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.22.0 +v0.23.0 From ee49355bb7274a463c2f5b0e76fa44220fe061ce Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 28 Jul 2022 16:44:58 +0300 Subject: [PATCH 291/548] [#179] Expose pool metrics Signed-off-by: Denis Kirillov --- app.go | 2 +- metrics/metrics.go | 161 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 160 insertions(+), 3 deletions(-) diff --git a/app.go b/app.go index bd9dd1d..56cf01a 100644 --- a/app.go +++ b/app.go @@ -165,7 +165,7 @@ func newApp(ctx context.Context, opt ...Option) App { } if a.cfg.GetBool(cfgPrometheusEnabled) { - a.metrics = metrics.NewGateMetrics() + a.metrics = metrics.NewGateMetrics(a.pool) } return a diff --git a/metrics/metrics.go b/metrics/metrics.go index 480cd81..9c79189 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -3,6 +3,7 @@ package metrics import ( "net/http" + "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/zap" @@ -11,23 +12,54 @@ import ( const ( namespace = "neofs_http_gw" stateSubsystem = "state" + poolSubsystem = "pool" + + methodGetBalance = "get_balance" + methodPutContainer = "put_container" + methodGetContainer = "get_container" + methodListContainer = "list_container" + methodDeleteContainer = "delete_container" + methodGetContainerEacl = "get_container_eacl" + methodSetContainerEacl = "set_container_eacl" + methodEndpointInfo = "endpoint_info" + methodNetworkInfo = "network_info" + methodPutObject = "put_object" + methodDeleteObject = "delete_object" + methodGetObject = "get_object" + methodHeadObject = "head_object" + methodRangeObject = "range_object" + methodCreateSession = "create_session" ) type GateMetrics struct { stateMetrics + poolMetricsCollector } type stateMetrics struct { healthCheck prometheus.Gauge } +type poolMetricsCollector struct { + pool *pool.Pool + overallErrors prometheus.Counter + overallNodeErrors *prometheus.CounterVec + overallNodeRequests *prometheus.CounterVec + currentErrors *prometheus.GaugeVec + requestDuration *prometheus.GaugeVec +} + // NewGateMetrics creates new metrics for http gate. -func NewGateMetrics() *GateMetrics { +func NewGateMetrics(p *pool.Pool) *GateMetrics { stateMetric := newStateMetrics() stateMetric.register() + poolMetric := newPoolMetricsCollector(p) + poolMetric.register() + return &GateMetrics{ - stateMetrics: *stateMetric, + stateMetrics: *stateMetric, + poolMetricsCollector: *poolMetric, } } @@ -50,6 +82,131 @@ func (m stateMetrics) SetHealth(s int32) { m.healthCheck.Set(float64(s)) } +func newPoolMetricsCollector(p *pool.Pool) *poolMetricsCollector { + overallErrors := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: "overall_errors", + Help: "Total number of errors in pool", + }, + ) + + overallNodeErrors := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: "overall_node_errors", + Help: "Total number of errors for connection in pool", + }, + []string{ + "node", + }, + ) + + overallNodeRequests := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: "overall_node_requests", + Help: "Total number of requests to specific node in pool", + }, + []string{ + "node", + }, + ) + + currentErrors := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: "current_errors", + Help: "Number of errors on current connections that will be reset after the threshold", + }, + []string{ + "node", + }, + ) + + requestsDuration := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: "avg_request_duration", + Help: "Average request duration (in milliseconds) for specific method on node in pool", + }, + []string{ + "node", + "method", + }, + ) + + return &poolMetricsCollector{ + pool: p, + overallErrors: overallErrors, + overallNodeErrors: overallNodeErrors, + overallNodeRequests: overallNodeRequests, + currentErrors: currentErrors, + requestDuration: requestsDuration, + } +} + +func (m *poolMetricsCollector) Collect(ch chan<- prometheus.Metric) { + m.updateStatistic() + m.overallErrors.Collect(ch) + m.overallNodeErrors.Collect(ch) + m.overallNodeRequests.Collect(ch) + m.currentErrors.Collect(ch) + m.requestDuration.Collect(ch) +} + +func (m poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) { + m.overallErrors.Describe(descs) + m.overallNodeErrors.Describe(descs) + m.overallNodeRequests.Describe(descs) + m.currentErrors.Describe(descs) + m.requestDuration.Describe(descs) +} + +func (m *poolMetricsCollector) register() { + prometheus.MustRegister(m) +} + +func (m *poolMetricsCollector) updateStatistic() { + stat := m.pool.Statistic() + + m.currentErrors.Reset() + m.requestDuration.Reset() + + for _, node := range stat.Nodes() { + m.overallNodeErrors.WithLabelValues(node.Address()).Add(float64(node.OverallErrors())) + m.overallNodeRequests.WithLabelValues(node.Address()).Add(float64(node.Requests())) + + m.currentErrors.WithLabelValues(node.Address()).Set(float64(node.CurrentErrors())) + m.updateRequestsDuration(node) + } + + m.overallErrors.Add(float64(stat.OverallErrors())) +} + +func (m *poolMetricsCollector) updateRequestsDuration(node pool.NodeStatistic) { + m.requestDuration.WithLabelValues(node.Address(), methodGetBalance).Set(float64(node.AverageGetBalance().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodPutContainer).Set(float64(node.AveragePutContainer().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodGetContainer).Set(float64(node.AverageGetContainer().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodListContainer).Set(float64(node.AverageListContainer().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodDeleteContainer).Set(float64(node.AverageDeleteContainer().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodGetContainerEacl).Set(float64(node.AverageGetContainerEACL().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodSetContainerEacl).Set(float64(node.AverageSetContainerEACL().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodEndpointInfo).Set(float64(node.AverageEndpointInfo().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodNetworkInfo).Set(float64(node.AverageNetworkInfo().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodPutObject).Set(float64(node.AveragePutObject().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodDeleteObject).Set(float64(node.AverageDeleteObject().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodGetObject).Set(float64(node.AverageGetObject().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodHeadObject).Set(float64(node.AverageHeadObject().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodRangeObject).Set(float64(node.AverageRangeObject().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodCreateSession).Set(float64(node.AverageCreateSession().Milliseconds())) +} + // NewPrometheusService creates a new service for gathering prometheus metrics. func NewPrometheusService(log *zap.Logger, cfg Config) *Service { if log == nil { From 9d4049cfe39418ec6227e4608728bbc0b2054cf3 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Tue, 2 Aug 2022 00:47:20 +0400 Subject: [PATCH 292/548] [#186] Don't print env vars with empty values Signed-off-by: Angira Kekteeva --- settings.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/settings.go b/settings.go index c6ad084..52de560 100644 --- a/settings.go +++ b/settings.go @@ -182,8 +182,13 @@ func settings() *viper.Viper { continue } + defaultValue := v.GetString(keys[i]) + if len(defaultValue) == 0 { + continue + } + k := strings.Replace(keys[i], ".", "_", -1) - fmt.Printf("%s_%s = %v\n", Prefix, strings.ToUpper(k), v.Get(keys[i])) + fmt.Printf("%s_%s = %s\n", Prefix, strings.ToUpper(k), defaultValue) } fmt.Println() From d8b04223d48d04eb9841c1257569b379f4d5f6b1 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Tue, 2 Aug 2022 00:52:07 +0400 Subject: [PATCH 293/548] [#186] Improve wallet params Signed-off-by: Angira Kekteeva --- app.go | 10 ++-------- settings.go | 8 ++++++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/app.go b/app.go index 56cf01a..ee979c4 100644 --- a/app.go +++ b/app.go @@ -181,10 +181,7 @@ func remove(list []string, element string) []string { } func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) { - walletPath := a.cfg.GetString(cmdWallet) - if len(walletPath) == 0 { - walletPath = a.cfg.GetString(cfgWalletPath) - } + walletPath := a.cfg.GetString(cfgWalletPath) if len(walletPath) == 0 { a.log.Info("no wallet path specified, creating ephemeral key automatically for this run") @@ -205,10 +202,7 @@ func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) { password = &pwd } - address := a.cfg.GetString(cmdAddress) - if len(address) == 0 { - address = a.cfg.GetString(cfgWalletAddress) - } + address := a.cfg.GetString(cfgWalletAddress) return getKeyFromWallet(w, address, password) } diff --git a/settings.go b/settings.go index 52de560..c9e6648 100644 --- a/settings.go +++ b/settings.go @@ -154,6 +154,14 @@ func settings() *viper.Viper { panic(err) } + if err := v.BindPFlag(cfgWalletPath, flags.Lookup(cmdWallet)); err != nil { + panic(err) + } + + if err := v.BindPFlag(cfgWalletAddress, flags.Lookup(cmdAddress)); err != nil { + panic(err) + } + if err := v.BindPFlags(flags); err != nil { panic(err) } From 6789dbc6a92ba88373de5869f4234819e55b2f75 Mon Sep 17 00:00:00 2001 From: anikeev-yadro Date: Thu, 11 Aug 2022 15:49:48 +0300 Subject: [PATCH 294/548] [#190] Filter version tags for version calculation Signed-off-by: anikeev-yadro --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c9d9be4..d0b8f43 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ #!/usr/bin/make -f REPO ?= $(shell go list -m) -VERSION ?= $(shell git describe --tags --dirty --always 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") +VERSION ?= $(shell git describe --tags --match "v*" --dirty --always 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") GO_VERSION ?= 1.17 LINT_VERSION ?= 1.46.2 BUILD ?= $(shell date -u --iso=seconds) From 1bd31a9e1df74a06699a04217aed4b7430e1e679 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 12 Aug 2022 12:54:19 +0300 Subject: [PATCH 295/548] [#191] Fix lint issues Signed-off-by: Denis Kirillov --- main.go | 6 +++--- uploader/multipart/multipart.go | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index b54fd67..5b4d188 100644 --- a/main.go +++ b/main.go @@ -26,9 +26,9 @@ func main() { // Panics on failure. // // Logger is built from zap's production logging configuration with: -// * parameterized level (debug by default) -// * console encoding -// * ISO8601 time encoding +// - parameterized level (debug by default) +// - console encoding +// - ISO8601 time encoding // // Logger records a stack trace for all messages at or above fatal level. // diff --git a/uploader/multipart/multipart.go b/uploader/multipart/multipart.go index 531b83f..69e6719 100644 --- a/uploader/multipart/multipart.go +++ b/uploader/multipart/multipart.go @@ -413,7 +413,8 @@ func (r *Reader) isBoundaryDelimiterLine(line []byte) (ret bool) { // skipLWSPChar returns b with leading spaces and tabs removed. // RFC 822 defines: -// LWSP-char = SPACE / HTAB +// +// LWSP-char = SPACE / HTAB func skipLWSPChar(b []byte) []byte { for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') { b = b[1:] From 035f043da6a3a01d8e9a5bb8019c8ded2eaec9f1 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 12 Aug 2022 18:30:23 +0300 Subject: [PATCH 296/548] [#189] Mention NNS in README Signed-off-by: Denis Kirillov --- README.md | 79 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index f98b936..8d03a43 100644 --- a/README.md +++ b/README.md @@ -188,29 +188,65 @@ and upload objects with it, but deleting, searching, managing ACLs, creating containers and other activities are not supported and not planned to be supported. -**Note:** in all download/upload routes you can use container name instead of it's id (`$CID`), but resolvers must be configured properly (see [configs](./config) for examples). - ### Preparation Before uploading or downloading a file make sure you have a prepared container. You can create it with instructions below. -Also ,in case of downloading, you need to have a file inside a container. +Also, in case of downloading, you need to have a file inside a container. + +### NNS + +In all download/upload routes you can use container name instead of its id (`$CID`). + +Steps to start using name resolving: + +1. Enable NNS resolving in config (`rpc_endpoint` must be a valid neo rpc node, see [configs](./config) for other examples): + +```yaml +rpc_endpoint: http://morph-chain.neofs.devenv:30333 +resolve_order: + - nns +``` + +2. Make sure your container is registered in NNS contract. If you use [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env) +you can check if your container (e.g. with `container-name` name) is registered in NNS: + +```shell +$ curl -s --data '{"id":1,"jsonrpc":"2.0","method":"getcontractstate","params":[1]}' \ + http://morph-chain.neofs.devenv:30333 | jq -r '.result.hash' + +0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 + +$ docker exec -it morph_chain neo-go \ + contract testinvokefunction \ + -r http://morph-chain.neofs.devenv:30333 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 \ + resolve string:container-name.container int:16 \ + | jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \ + | base64 -d && echo + +7f3vvkw4iTiS5ZZbu5BQXEmJtETWbi3uUjLNaSs29xrL +``` + +3. Use container name instead of its `$CID`. For example: + +```shell +$ curl http://localhost:8082/get_by_attribute/container-name/FileName/object-name +``` #### Create a container You can create a container via [neofs-cli](https://github.com/nspcc-dev/neofs-node/releases): ``` -$ neofs-cli -r $NEOFS_NODE -k $KEY container create --policy $POLICY --basic-acl $ACL +$ neofs-cli -r $NEOFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL ``` -where `$KEY` can be a path to a private key file (as raw bytes), a hex string or -a (unencrypted) WIF string, +where `$WALLET` is a path to user wallet, `$ACL` -- hex encoded basic ACL value or keywords 'private, 'public-read', 'public-read-write' and `$POLICY` -- QL-encoded or JSON-encoded placement policy or path to file with it For example: ``` -$ neofs-cli -r 192.168.130.72:8080 -k 6PYLKJhiSub5imt6WCVy6Quxtd9xu176omev1vWYovzkAQCTSQabAAQXii container create --policy "REP 3" --basic-acl public --await +$ neofs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy "REP 3" --basic-acl public --await ``` If you have launched nodes via [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env), @@ -229,7 +265,7 @@ where For example: ``` -$ neofs-cli -r 192.168.130.72:8080 -k 6PYLKJhiSub5imt6WCVy6Quxtd9xu176omev1vWYovzkAQCTSQabAAQXii object put --file cat.png --cid DPL2tpRiuDNmoTj5KZjD1nzDuCS8tVcxa7hsvSLDWpVM --attributes img_type=cat,my_attr=cute +$ neofs-cli -r 192.168.130.72:8080 -w ./wallet.json object put --file cat.png --cid Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ --attributes img_type=cat,my_attr=cute ``` @@ -242,15 +278,21 @@ The following requests support GET/HEAD methods. ##### By IDs Basic downloading involves container ID and object ID and is done via GET -requests to `/get/$CID/$OID` path, where `$CID` is a container ID, +requests to `/get/$CID/$OID` path, where `$CID` is a container ID or its name if NNS is enabled, `$OID` is an object's (i.e. your file's) ID. - For example: +For example: -``` +```shell $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m8PtaoricLouCn5zE8hAFr3gZEBDCZFe9BEgVJTSocY ``` +or if container has a name: + +```shell +$ wget http://localhost:8082/get/container-name/2m8PtaoricLouCn5zE8hAFr3gZEBDCZFe9BEgVJTSocY +``` + ##### By attributes There is also more complex interface provided for attribute-based downloads, it's usually used to retrieve files by their names, but any other attribute @@ -259,7 +301,7 @@ can be used as well. The generic syntax for it looks like this: ```/get_by_attribute/$CID/$ATTRIBUTE_NAME/$ATTRIBUTE_VALUE``` where -`$CID` is a container ID, +`$CID` is a container ID or its name if NNS is enabled, `$ATTRIBUTE_NAME` is the name of the attribute we want to use, `$ATTRIBUTE_VALUE` is the value of this attribute that the target object should have. @@ -333,7 +375,7 @@ set of reply headers generated using the following rules: ### Uploading -You can POST files to `/upload/$CID` path where `$CID` is a container ID. The +You can POST files to `/upload/$CID` path where `$CID` is a container ID or its name if NNS is enabled. The request must contain multipart form with mandatory `filename` parameter. Only one part in multipart form will be processed, so to upload another file just issue a new POST request. @@ -471,7 +513,7 @@ Now, we can form a Bearer token (10000 is liftetime expiration in epoch) and sav Next, sign it with the container owner key: ``` -$ neofs-cli util sign bearer-token --from bearer.json --to signed.json -k KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr +$ neofs-cli util sign bearer-token --from bearer.json --to signed.json -w ./wallet.json ``` Encoding to base64 to use via the header: ``` @@ -498,12 +540,12 @@ For the token to work correctly, you need to create a container with a basic ACL For example: ``` -$ neofs-cli --key KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr --basic-acl 0x0FFFCFFF -r 192.168.130.72:8080 container create --policy "REP 3" --await +$ neofs-cli -w ./wallet.json --basic-acl 0x0FFFCFFF -r 192.168.130.72:8080 container create --policy "REP 3" --await ``` To deny access to a container without a token, set the eACL rules: ``` -$ neofs-cli --key KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr -r 192.168.130.72:8080 container set-eacl --table eacl.json --await --cid BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K +$ neofs-cli -w ./wallet.json -r 192.168.130.72:8080 container set-eacl --table eacl.json --await --cid BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K ``` File **eacl.json**: @@ -534,8 +576,9 @@ File **eacl.json**: ### Metrics and Pprof -If enabled, Prometheus metrics are available at `/metrics/` path and Pprof at -`/debug/pprof`. +If enabled, Prometheus metrics are available at `localhost:8084` endpoint +and Pprof at `localhost:8083/debug/pprof` by default. Host and port can be configured. +See [configuration](./docs/gate-configuration.md). ## Credits From 9ebcb6b9645f59b0250fc17831cb91c4142cd514 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 22 Aug 2022 18:00:13 +0300 Subject: [PATCH 297/548] [#188] Update SDK to avoid panic on go1.19 Signed-off-by: Denis Kirillov --- downloader/download.go | 2 +- downloader/head.go | 8 ++++++-- go.mod | 12 ++++++------ go.sum | 17 ++++++++++------- uploader/upload.go | 4 ++-- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index f43d93d..e34e3b7 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -368,7 +368,7 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.P f(*d.newRequest(c, log), d.pool, addrObj) } -func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) (*pool.ResObjectSearch, error) { +func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) (pool.ResObjectSearch, error) { filters := object.NewSearchFilters() filters.AddRootFilter() filters.AddFilter(key, val, op) diff --git a/downloader/head.go b/downloader/head.go index bcddd08..eafed21 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -72,7 +72,7 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { } } - idsToResponse(&r.Response, obj) + idsToResponse(&r.Response, &obj) if len(contentType) == 0 { contentType, _, err = readContentType(obj.PayloadSize(), func(sz uint64) (io.Reader, error) { @@ -83,7 +83,11 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { prmRange.UseBearer(*btoken) } - return clnt.ObjectRange(r.appCtx, prmRange) + resObj, err := clnt.ObjectRange(r.appCtx, prmRange) + if err != nil { + return nil, err + } + return &resObj, nil }) if err != nil && err != io.EOF { r.handleNeoFSErr(err, start) diff --git a/go.mod b/go.mod index 89abe20..ba8b37d 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,10 @@ go 1.17 require ( github.com/fasthttp/router v1.4.1 - github.com/nspcc-dev/neo-go v0.99.1-pre.0.20220714084516-54849ef3e58e - github.com/nspcc-dev/neofs-api-go/v2 v2.13.0 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.5.0.20220725101411-90f1cc7a1a77 + github.com/nspcc-dev/neo-go v0.99.1 + github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220818094951-98db3fa28419 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220824140410-7537fa0decc5 github.com/prometheus/client_golang v1.11.0 - github.com/prometheus/common v0.30.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 @@ -56,9 +55,9 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect github.com/nspcc-dev/hrw v1.0.9 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220713145417-4f184498bc42 // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220727202624-6c7a401f776a // indirect github.com/nspcc-dev/neofs-contract v0.15.3 // indirect - github.com/nspcc-dev/neofs-crypto v0.3.0 // indirect + github.com/nspcc-dev/neofs-crypto v0.4.0 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/nspcc-dev/tzhash v1.6.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -68,6 +67,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.30.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect diff --git a/go.sum b/go.sum index ad66064..5929f56 100644 --- a/go.sum +++ b/go.sum @@ -740,24 +740,27 @@ github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= -github.com/nspcc-dev/neo-go v0.99.1-pre.0.20220714084516-54849ef3e58e h1:sjl8sniYkjoOsD8F+wzkpRfm9RzZSLQlh5Z/SMyf4A8= github.com/nspcc-dev/neo-go v0.99.1-pre.0.20220714084516-54849ef3e58e/go.mod h1:/y5Sl8p3YheTygriBtCCMWKkDOek8HcvSo5ds2rJtKI= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220713145417-4f184498bc42 h1:Krhg2cD5mqvC+lin7irw6hj0M+x4ZOZrRGzrZQB+wcQ= +github.com/nspcc-dev/neo-go v0.99.1 h1:/dlDVVCqlmazusIu+Emd4TlwxjIOpNNU5ob7bwUH6Qk= +github.com/nspcc-dev/neo-go v0.99.1/go.mod h1:wvKMQXlE/M3eqtN0KIi1MjkEVTvYIJRrOMnJtqEkCxs= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220713145417-4f184498bc42/go.mod h1:QBE0I30F2kOAISNpT5oks82yF4wkkUq3SCfI3Hqgx/Y= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220727202624-6c7a401f776a h1:ND5lYt9/cybrbS6/DVX0398a4ilt4/xE/QtEaOzJSGY= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220727202624-6c7a401f776a/go.mod h1:QBE0I30F2kOAISNpT5oks82yF4wkkUq3SCfI3Hqgx/Y= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.13.0 h1:7BcBiSjmWqJx0SPFlGRYt9ZFbMjucRIz9+Mv8UBLeq8= -github.com/nspcc-dev/neofs-api-go/v2 v2.13.0/go.mod h1:73j09Xa7I2zQbM3HCvAHnDHPYiiWnEHa1d6Z6RDMBLU= +github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220818094951-98db3fa28419 h1:NFj8D0UYfHzvC+gIhK7yIUOyEcZi21caQsuYVbJGrVU= +github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220818094951-98db3fa28419/go.mod h1:NAaDfOnFWIbAFkTj7pNQ+cknVue0JbdeRT9QQaeBCEY= github.com/nspcc-dev/neofs-contract v0.15.3 h1:7+NwyTtxFAnIevz0hR/XxQf6R2Ej2scjVR2bnnJnhBM= github.com/nspcc-dev/neofs-contract v0.15.3/go.mod h1:BXVZUZUJxrmmDETglXHI8+5DSgn84B9y5DoSWqEjYCs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= +github.com/nspcc-dev/neofs-crypto v0.4.0 h1:5LlrUAM5O0k1+sH/sktBtrgfWtq1pgpDs09fZo+KYi4= +github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.5.0.20220725101411-90f1cc7a1a77 h1:C8QL3pUN/QZ7OdXONEV2FQ+JamXoBbovvZiylWOCfBo= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.5.0.20220725101411-90f1cc7a1a77/go.mod h1:39SbCo+QUI0WJS47VsW4SCWhOwnJNpQSy8rGEG/b5vc= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220824140410-7537fa0decc5 h1:BkjgenUQeXnMnbvdpyn8tOoxZmaeKOsN6EEq8wF9Xnw= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220824140410-7537fa0decc5/go.mod h1:BQPxy2sOURSegZJVnrwYDagPtKI3H1Ctc7e0yS0xHvk= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/uploader/upload.go b/uploader/upload.go index d6b06c8..4c1172c 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -59,7 +59,7 @@ func New(ctx context.Context, params *utils.AppParams, enableDefaultTimestamp bo func (u *Uploader) Upload(c *fasthttp.RequestCtx) { var ( file MultipartFile - idObj *oid.ID + idObj oid.ID addr oid.Address scid, _ = c.UserValue("cid").(string) log = u.log.With(zap.String("cid", scid)) @@ -157,7 +157,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { return } - addr.SetObject(*idObj) + addr.SetObject(idObj) addr.SetContainer(*idCnr) // Try to return the response, otherwise, if something went wrong, throw an error. From 4348f7253a99f924ac414a5f7fed99a31fac182c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 22 Aug 2022 18:29:43 +0300 Subject: [PATCH 298/548] [#188] Fix tests Signed-off-by: Denis Kirillov --- integration_test.go | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/integration_test.go b/integration_test.go index 4e76163..9f55486 100644 --- a/integration_test.go +++ b/integration_test.go @@ -38,6 +38,8 @@ type putResponse struct { const ( testContainerName = "friendly" versionWithNativeNames = "0.27.5" + testListenAddress = "localhost:8082" + testHost = "http://" + testListenAddress ) func TestIntegration(t *testing.T) { @@ -87,17 +89,17 @@ func runServer() context.CancelFunc { return cancel } -func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID *cid.ID, version string) { - url := "http://localhost:8082/upload/" + CID.String() +func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID, version string) { + url := testHost + "/upload/" + CID.String() makePutRequestAndCheck(ctx, t, p, CID, url) if version >= versionWithNativeNames { - url = "http://localhost:8082/upload/" + testContainerName + url = testHost + "/upload/" + testContainerName makePutRequestAndCheck(ctx, t, p, CID, url) } } -func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.ID, url string) { +func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID cid.ID, url string) { content := "content of file" keyAttr, valAttr := "User-Attribute", "user value" attributes := map[string]string{ @@ -142,13 +144,13 @@ func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnr err = cnrID.DecodeString(addr.CID) require.NoError(t, err) - id := new(oid.ID) + var id oid.ID err = id.DecodeString(addr.OID) require.NoError(t, err) var objectAddress oid.Address - objectAddress.SetContainer(*cnrID) - objectAddress.SetObject(*id) + objectAddress.SetContainer(cnrID) + objectAddress.SetObject(id) payload := bytes.NewBuffer(nil) @@ -168,7 +170,7 @@ func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnr } } -func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID *cid.ID, version string) { +func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { content := "content of file" attributes := map[string]string{ "some-attr": "some-get-value", @@ -176,12 +178,12 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID id := putObject(ctx, t, clientPool, ownerID, CID, content, attributes) - resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + id.String()) + resp, err := http.Get(testHost + "/get/" + CID.String() + "/" + id.String()) require.NoError(t, err) checkGetResponse(t, resp, content, attributes) if version >= versionWithNativeNames { - resp, err = http.Get("http://localhost:8082/get/" + testContainerName + "/" + id.String()) + resp, err = http.Get(testHost + "/get/" + testContainerName + "/" + id.String()) require.NoError(t, err) checkGetResponse(t, resp, content, attributes) } @@ -217,7 +219,7 @@ func checkGetByAttrResponse(t *testing.T, resp *http.Response, content string, a } } -func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID *cid.ID, version string) { +func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { keyAttr, valAttr := "some-attr", "some-get-by-attr-value" content := "content of file" attributes := map[string]string{keyAttr: valAttr} @@ -230,18 +232,18 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID "x-container-id": CID.String(), } - resp, err := http.Get("http://localhost:8082/get_by_attribute/" + CID.String() + "/" + keyAttr + "/" + valAttr) + resp, err := http.Get(testHost + "/get_by_attribute/" + CID.String() + "/" + keyAttr + "/" + valAttr) require.NoError(t, err) checkGetByAttrResponse(t, resp, content, expectedAttr) if version >= versionWithNativeNames { - resp, err = http.Get("http://localhost:8082/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr) + resp, err = http.Get(testHost + "/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr) require.NoError(t, err) checkGetByAttrResponse(t, resp, content, expectedAttr) } } -func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID *cid.ID, version string) { +func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"} contents := []string{"content of file1", "content of file2"} attributes1 := map[string]string{attributeFilePath: names[0]} @@ -250,11 +252,11 @@ func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID us putObject(ctx, t, clientPool, ownerID, CID, contents[0], attributes1) putObject(ctx, t, clientPool, ownerID, CID, contents[1], attributes2) - baseURL := "http://localhost:8082/zip/" + CID.String() + baseURL := testHost + "/zip/" + CID.String() makeZipTest(t, baseURL, names, contents) if version >= versionWithNativeNames { - baseURL = "http://localhost:8082/zip/" + testContainerName + baseURL = testHost + "/zip/" + testContainerName makeZipTest(t, baseURL, names, contents) } } @@ -332,6 +334,7 @@ func getDefaultConfig() *viper.Viper { v.SetDefault(cfgPeers+".0.priority", 1) v.SetDefault(cfgRPCEndpoint, "http://localhost:30333") + v.SetDefault(cfgListenAddress, testListenAddress) return v } @@ -350,7 +353,7 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool return clientPool } -func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, version string) (*cid.ID, error) { +func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, version string) (cid.ID, error) { var policy netmap.PlacementPolicy err := policy.DecodeString("REP 1") require.NoError(t, err) @@ -379,16 +382,16 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o CID, err := clientPool.PutContainer(ctx, prm) if err != nil { - return nil, err + return cid.ID{}, err } fmt.Println(CID.String()) return CID, err } -func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID *cid.ID, content string, attributes map[string]string) *oid.ID { +func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, content string, attributes map[string]string) oid.ID { obj := object.New() - obj.SetContainerID(*CID) + obj.SetContainerID(CID) obj.SetOwnerID(&ownerID) var attrs []object.Attribute From e0ab9294103bab4a6cf96c5500a5430069d0fce7 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 25 Aug 2022 09:36:38 +0300 Subject: [PATCH 299/548] [#194] Use gauge instead of counter for metrics Signed-off-by: Denis Kirillov --- metrics/metrics.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index 9c79189..52b5e88 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -42,9 +42,9 @@ type stateMetrics struct { type poolMetricsCollector struct { pool *pool.Pool - overallErrors prometheus.Counter - overallNodeErrors *prometheus.CounterVec - overallNodeRequests *prometheus.CounterVec + overallErrors prometheus.Gauge + overallNodeErrors *prometheus.GaugeVec + overallNodeRequests *prometheus.GaugeVec currentErrors *prometheus.GaugeVec requestDuration *prometheus.GaugeVec } @@ -83,8 +83,8 @@ func (m stateMetrics) SetHealth(s int32) { } func newPoolMetricsCollector(p *pool.Pool) *poolMetricsCollector { - overallErrors := prometheus.NewCounter( - prometheus.CounterOpts{ + overallErrors := prometheus.NewGauge( + prometheus.GaugeOpts{ Namespace: namespace, Subsystem: poolSubsystem, Name: "overall_errors", @@ -92,8 +92,8 @@ func newPoolMetricsCollector(p *pool.Pool) *poolMetricsCollector { }, ) - overallNodeErrors := prometheus.NewCounterVec( - prometheus.CounterOpts{ + overallNodeErrors := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ Namespace: namespace, Subsystem: poolSubsystem, Name: "overall_node_errors", @@ -104,8 +104,8 @@ func newPoolMetricsCollector(p *pool.Pool) *poolMetricsCollector { }, ) - overallNodeRequests := prometheus.NewCounterVec( - prometheus.CounterOpts{ + overallNodeRequests := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ Namespace: namespace, Subsystem: poolSubsystem, Name: "overall_node_requests", @@ -175,18 +175,20 @@ func (m *poolMetricsCollector) register() { func (m *poolMetricsCollector) updateStatistic() { stat := m.pool.Statistic() + m.overallNodeErrors.Reset() + m.overallNodeRequests.Reset() m.currentErrors.Reset() m.requestDuration.Reset() for _, node := range stat.Nodes() { - m.overallNodeErrors.WithLabelValues(node.Address()).Add(float64(node.OverallErrors())) - m.overallNodeRequests.WithLabelValues(node.Address()).Add(float64(node.Requests())) + m.overallNodeErrors.WithLabelValues(node.Address()).Set(float64(node.OverallErrors())) + m.overallNodeRequests.WithLabelValues(node.Address()).Set(float64(node.Requests())) m.currentErrors.WithLabelValues(node.Address()).Set(float64(node.CurrentErrors())) m.updateRequestsDuration(node) } - m.overallErrors.Add(float64(stat.OverallErrors())) + m.overallErrors.Set(float64(stat.OverallErrors())) } func (m *poolMetricsCollector) updateRequestsDuration(node pool.NodeStatistic) { From 1c51979a03acc56ec6077cf187810c025fd3a3ee Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 29 Aug 2022 10:48:16 +0300 Subject: [PATCH 300/548] [#196] Update sdk Signed-off-by: Denis Kirillov --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ba8b37d..48814d4 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.17 require ( github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.99.1 - github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220818094951-98db3fa28419 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220824140410-7537fa0decc5 + github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220827080658-9e17cdfc7647 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220829114550-ee92df32032e github.com/prometheus/client_golang v1.11.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 diff --git a/go.sum b/go.sum index 5929f56..592a895 100644 --- a/go.sum +++ b/go.sum @@ -748,8 +748,8 @@ github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220727202624-6c7a401f776a h1:ND github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220727202624-6c7a401f776a/go.mod h1:QBE0I30F2kOAISNpT5oks82yF4wkkUq3SCfI3Hqgx/Y= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220818094951-98db3fa28419 h1:NFj8D0UYfHzvC+gIhK7yIUOyEcZi21caQsuYVbJGrVU= -github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220818094951-98db3fa28419/go.mod h1:NAaDfOnFWIbAFkTj7pNQ+cknVue0JbdeRT9QQaeBCEY= +github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220827080658-9e17cdfc7647 h1:00JDCprdZG15n8z58EaFvJlmvZ4uESUii8MSzk869JM= +github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220827080658-9e17cdfc7647/go.mod h1:NAaDfOnFWIbAFkTj7pNQ+cknVue0JbdeRT9QQaeBCEY= github.com/nspcc-dev/neofs-contract v0.15.3 h1:7+NwyTtxFAnIevz0hR/XxQf6R2Ej2scjVR2bnnJnhBM= github.com/nspcc-dev/neofs-contract v0.15.3/go.mod h1:BXVZUZUJxrmmDETglXHI8+5DSgn84B9y5DoSWqEjYCs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= @@ -759,8 +759,8 @@ github.com/nspcc-dev/neofs-crypto v0.4.0 h1:5LlrUAM5O0k1+sH/sktBtrgfWtq1pgpDs09f github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220824140410-7537fa0decc5 h1:BkjgenUQeXnMnbvdpyn8tOoxZmaeKOsN6EEq8wF9Xnw= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220824140410-7537fa0decc5/go.mod h1:BQPxy2sOURSegZJVnrwYDagPtKI3H1Ctc7e0yS0xHvk= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220829114550-ee92df32032e h1:4bw+na7QGjEHnZeNGhW31+l/MQ6N5NZ1LxCbldY0Rgo= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220829114550-ee92df32032e/go.mod h1:zANAq4ZCA1YFX+DSzP/ydBv/Krj5sdkK5orpTISmZR4= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= From fdfcc7a47447a1363f9d0da29821822e83216216 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 30 Aug 2022 15:58:31 +0300 Subject: [PATCH 301/548] [#197] Update go version to 1.19 Signed-off-by: Denis Kirillov --- .github/workflows/builds.yml | 4 ++-- .github/workflows/tests.yml | 4 ++-- Dockerfile | 2 +- Makefile | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 8c89e07..c19d45d 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -21,7 +21,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: 1.17 + go-version: 1.19 - name: Restore Go modules from cache uses: actions/cache@v2 @@ -54,7 +54,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.17 + go-version: 1.19 - name: Restore Go modules from cache uses: actions/cache@v2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0874b72..ca32b75 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.17 + go-version: 1.19 - name: Restore Go modules from cache uses: actions/cache@v2 @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - go_versions: [ '1.17', '1.18' ] + go_versions: [ '1.18', '1.19' ] fail-fast: false steps: - uses: actions/checkout@v2 diff --git a/Dockerfile b/Dockerfile index 015e4b5..b3ac08f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17-alpine as basebuilder +FROM golang:1.19-alpine as basebuilder RUN apk add --update make bash ca-certificates FROM basebuilder as builder diff --git a/Makefile b/Makefile index d0b8f43..458e45c 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --match "v*" --dirty --always 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") -GO_VERSION ?= 1.17 +GO_VERSION ?= 1.19 LINT_VERSION ?= 1.46.2 BUILD ?= $(shell date -u --iso=seconds) From 4dcdb8ef028349e56ab8bf1c313b164a4da07f56 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 30 Aug 2022 16:00:01 +0300 Subject: [PATCH 302/548] [#197] Update lint to v1.49.0 Signed-off-by: Denis Kirillov --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 458e45c..6f1fcc8 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --match "v*" --dirty --always 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") GO_VERSION ?= 1.19 -LINT_VERSION ?= 1.46.2 +LINT_VERSION ?= 1.49.0 BUILD ?= $(shell date -u --iso=seconds) HUB_IMAGE ?= nspccdev/neofs-http-gw From a1052c04db16252d374af465dfd38aaa30350bbf Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 1 Sep 2022 12:21:27 +0300 Subject: [PATCH 303/548] [#198] Fix expiration epoch calculation Previous implementation does not provide 'at least' lifetime guarantee. Signed-off-by: Alex Vanin --- uploader/filter.go | 18 +++++++++++++++--- uploader/filter_test.go | 20 ++++++++++++++++++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/uploader/filter.go b/uploader/filter.go index 0152920..e9e9395 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -3,6 +3,7 @@ package uploader import ( "bytes" "fmt" + "math" "strconv" "time" @@ -121,7 +122,18 @@ func prepareExpirationHeader(headers map[string]string, epochDurations *epochDur } func updateExpirationHeader(headers map[string]string, durations *epochDurations, expDuration time.Duration) { - epochDuration := durations.msPerBlock * int64(durations.blockPerEpoch) - numEpoch := expDuration.Milliseconds() / epochDuration - headers[object.SysAttributeExpEpoch] = strconv.FormatInt(int64(durations.currentEpoch)+numEpoch, 10) + epochDuration := uint64(durations.msPerBlock) * durations.blockPerEpoch + currentEpoch := durations.currentEpoch + numEpoch := uint64(expDuration.Milliseconds()) / epochDuration + + if uint64(expDuration.Milliseconds())%epochDuration != 0 { + numEpoch++ + } + + expirationEpoch := uint64(math.MaxUint64) + if numEpoch < math.MaxUint64-currentEpoch { + expirationEpoch = currentEpoch + numEpoch + } + + headers[object.SysAttributeExpEpoch] = strconv.FormatUint(expirationEpoch, 10) } diff --git a/uploader/filter_test.go b/uploader/filter_test.go index 913f076..f1e300d 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -1,6 +1,7 @@ package uploader import ( + "math" "strconv" "testing" "time" @@ -52,8 +53,13 @@ func TestPrepareExpirationHeader(t *testing.T) { blockPerEpoch: 101, } - epochPerDay := (24 * time.Hour).Milliseconds() / int64(defaultDurations.blockPerEpoch) / defaultDurations.msPerBlock - defaultExpEpoch := strconv.FormatInt(int64(defaultDurations.currentEpoch)+epochPerDay, 10) + msPerBlock := defaultDurations.blockPerEpoch * uint64(defaultDurations.msPerBlock) + epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock + if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 { + epochPerDay++ + } + + defaultExpEpoch := strconv.FormatUint(defaultDurations.currentEpoch+epochPerDay, 10) for _, tc := range []struct { name string @@ -130,6 +136,16 @@ func TestPrepareExpirationHeader(t *testing.T) { durations: defaultDurations, expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, }, + { + name: "valid max uint 64", + headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)}, + durations: &epochDurations{ + currentEpoch: math.MaxUint64 - 1, + msPerBlock: defaultDurations.msPerBlock, + blockPerEpoch: defaultDurations.blockPerEpoch, + }, + expected: map[string]string{object.SysAttributeExpEpoch: strconv.FormatUint(uint64(math.MaxUint64), 10)}, + }, { name: "invalid timestamp sec", headers: map[string]string{utils.ExpirationTimestampAttr: "abc"}, From 23df22fe3592dae84a06c7009ee7a2a4a405e06f Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 1 Sep 2022 16:02:21 +0300 Subject: [PATCH 304/548] [#198] Add unreleased section in changelog Signed-off-by: Alex Vanin --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fd90fc..b3bfb79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ This document outlines major changes between releases. +## Unreleased + +### Fixed +- Fix expiration epoch calculation (#198) + ## [0.23.0] - 2022-08-02 ### Added @@ -205,3 +210,4 @@ releases. [0.21.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.20.0...v0.21.0 [0.22.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.21.0...v0.22.0 [0.23.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.22.0...v0.23.0 +[Unreleased]: https://github.com/nspcc-dev/neofs-rest-gw/compare/v0.23.0...master From 00a08c1badeabf7730a23ed34df8a7492b81bf84 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 5 Sep 2022 12:02:08 +0300 Subject: [PATCH 305/548] [#199] Fix commit abbrev length in version Signed-off-by: Alex Vanin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6f1fcc8..d961a94 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ #!/usr/bin/make -f REPO ?= $(shell go list -m) -VERSION ?= $(shell git describe --tags --match "v*" --dirty --always 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") +VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") GO_VERSION ?= 1.19 LINT_VERSION ?= 1.49.0 BUILD ?= $(shell date -u --iso=seconds) From 56e089651de0c425a61acba45a144ff202f546cc Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 9 Sep 2022 12:20:27 +0300 Subject: [PATCH 306/548] [#202] Return go1.17 to CI tests Signed-off-by: Denis Kirillov --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ca32b75..5c8b409 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - go_versions: [ '1.18', '1.19' ] + go_versions: [ '1.17', '1.18', '1.19' ] fail-fast: false steps: - uses: actions/checkout@v2 From 23edd2273c5405ccde9bd58b5474705e798a2881 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 14 Sep 2022 10:45:39 +0300 Subject: [PATCH 307/548] Release v0.24.0 Signed-off-by: Denis Kirillov --- CHANGELOG.md | 17 +++++++++++++++-- VERSION | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3bfb79..c35e5e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,22 @@ This document outlines major changes between releases. -## Unreleased +## [Unreleased] + +## [0.24.0] - 2022-09-14 ### Fixed - Fix expiration epoch calculation (#198) +- Fix panic on go1.19 (#188) + +### Added +- Exposure of pool metrics (#179, #194) + +## Changed +- Help doesn't print empty parameters (#186) +- Update version calculation (#190, #199) +- Update neofs-sdk-go (#196) +- Update go version in CI and docker (#197, #202) ## [0.23.0] - 2022-08-02 @@ -210,4 +222,5 @@ releases. [0.21.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.20.0...v0.21.0 [0.22.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.21.0...v0.22.0 [0.23.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.22.0...v0.23.0 -[Unreleased]: https://github.com/nspcc-dev/neofs-rest-gw/compare/v0.23.0...master +[0.24.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.23.0...v0.24.0 +[Unreleased]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.24.0...master diff --git a/VERSION b/VERSION index 0c2a959..6897c00 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.23.0 +v0.24.0 From 6610420acd4ad1838467a1573b2c9339e443cd85 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 14 Sep 2022 17:17:01 +0300 Subject: [PATCH 308/548] [#204] Fix changelog section Signed-off-by: Denis Kirillov --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c35e5e2..28f7d25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ This document outlines major changes between releases. ### Added - Exposure of pool metrics (#179, #194) -## Changed +### Changed - Help doesn't print empty parameters (#186) - Update version calculation (#190, #199) - Update neofs-sdk-go (#196) From 25c9bc81fa40e25433f5ffdfc26be4b39c513d41 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 15 Sep 2022 19:03:49 +0300 Subject: [PATCH 309/548] [#205] Do not store builds in GitHub artifactory Signed-off-by: Alex Vanin --- .github/workflows/builds.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index c19d45d..d1f6b16 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -35,12 +35,6 @@ jobs: - name: Build CLI run: make - - name: Save binary - uses: actions/upload-artifact@v2 - with: - name: neofs-http-gw - path: bin/neofs-http-gw - build_image: needs: build_cli name: Build Docker image From 939f5f0c654d3ac0b50102b189ecd60f0beaa46c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 8 Sep 2022 17:57:22 +0300 Subject: [PATCH 310/548] [#200] Reload config level and metrics on SIGHUP Signed-off-by: Denis Kirillov --- app.go | 196 +++++++++++++++++++++++++++++++++----------- integration_test.go | 4 +- main.go | 63 ++------------ settings.go | 72 +++++++++++++++- 4 files changed, 226 insertions(+), 109 deletions(-) diff --git a/app.go b/app.go index ee979c4..6a86726 100644 --- a/app.go +++ b/app.go @@ -4,7 +4,11 @@ import ( "context" "crypto/ecdsa" "fmt" + "os" + "os/signal" "strconv" + "sync" + "syscall" "github.com/fasthttp/router" "github.com/nspcc-dev/neo-go/cli/flags" @@ -28,13 +32,15 @@ import ( type ( app struct { log *zap.Logger + logLevel zap.AtomicLevel pool *pool.Pool owner *user.ID cfg *viper.Viper webServer *fasthttp.Server webDone chan struct{} resolver *resolver.ContainerResolver - metrics GateMetricsProvider + metrics *gateMetrics + services []*metrics.Service } // App is an interface for the main gateway function. @@ -46,18 +52,26 @@ type ( // Option is an application option. Option func(a *app) + gateMetrics struct { + logger *zap.Logger + provider GateMetricsProvider + mu sync.RWMutex + enabled bool + } + GateMetricsProvider interface { SetHealth(int32) } ) // WithLogger returns Option to set a specific logger. -func WithLogger(l *zap.Logger) Option { +func WithLogger(l *zap.Logger, lvl zap.AtomicLevel) Option { return func(a *app) { if l == nil { return } a.log = l + a.logLevel = lvl } } @@ -164,13 +178,47 @@ func newApp(ctx context.Context, opt ...Option) App { a.log.Info("container resolver is disabled") } - if a.cfg.GetBool(cfgPrometheusEnabled) { - a.metrics = metrics.NewGateMetrics(a.pool) - } + a.initMetrics() return a } +func (a *app) initMetrics() { + gateMetricsProvider := metrics.NewGateMetrics(a.pool) + a.metrics = newGateMetrics(a.log, gateMetricsProvider, a.cfg.GetBool(cfgPrometheusEnabled)) +} + +func newGateMetrics(logger *zap.Logger, provider GateMetricsProvider, enabled bool) *gateMetrics { + if !enabled { + logger.Warn("metrics are disabled") + } + return &gateMetrics{ + logger: logger, + provider: provider, + } +} + +func (m *gateMetrics) SetEnabled(enabled bool) { + if !enabled { + m.logger.Warn("metrics are disabled") + } + + m.mu.Lock() + m.enabled = enabled + m.mu.Unlock() +} + +func (m *gateMetrics) SetHealth(status int32) { + m.mu.RLock() + if !m.enabled { + m.mu.RUnlock() + return + } + m.mu.RUnlock() + + m.provider.SetHealth(status) +} + func remove(list []string, element string) []string { for i, item := range list { if item == element { @@ -242,19 +290,110 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*ecds func (a *app) Wait() { a.log.Info("starting application", zap.String("app_name", "neofs-http-gw"), zap.String("version", Version)) - if a.metrics != nil { - a.metrics.SetHealth(1) - } + + a.setHealthStatus() <-a.webDone // wait for web-server to be stopped } +func (a *app) setHealthStatus() { + a.metrics.SetHealth(1) +} + func (a *app) Serve(ctx context.Context) { edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) uploadRoutes := uploader.New(ctx, a.AppParams(), edts) downloadSettings := downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)} downloadRoutes := downloader.New(ctx, a.AppParams(), downloadSettings) + // Configure router. + a.configureRouter(uploadRoutes, downloadRoutes) + + a.startServices() + + bind := a.cfg.GetString(cfgListenAddress) + tlsCertPath := a.cfg.GetString(cfgTLSCertificate) + tlsKeyPath := a.cfg.GetString(cfgTLSKey) + + go func() { + var err error + if tlsCertPath == "" && tlsKeyPath == "" { + a.log.Info("running web server", zap.String("address", bind)) + err = a.webServer.ListenAndServe(bind) + } else { + a.log.Info("running web server (TLS-enabled)", zap.String("address", bind)) + err = a.webServer.ListenAndServeTLS(bind, tlsCertPath, tlsKeyPath) + } + if err != nil { + a.log.Fatal("could not start server", zap.Error(err)) + } + }() + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGHUP) + +LOOP: + for { + select { + case <-ctx.Done(): + break LOOP + case <-sigs: + a.configReload() + } + } + + a.log.Info("shutting down web server", zap.Error(a.webServer.Shutdown())) + + a.stopServices() + + close(a.webDone) +} + +func (a *app) configReload() { + a.log.Info("SIGHUP config reload") + if !a.cfg.IsSet(cmdConfig) { + a.log.Warn("failed to reload config because it's missed") + return + } + if err := readConfig(a.cfg); err != nil { + a.log.Warn("failed to reload config", zap.Error(err)) + return + } + if lvl, err := getLogLevel(a.cfg); err != nil { + a.log.Warn("log level won't be updated", zap.Error(err)) + } else { + a.logLevel.SetLevel(lvl) + } + + a.stopServices() + a.startServices() + + a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled)) + a.setHealthStatus() +} + +func (a *app) startServices() { + pprofConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPprofEnabled), Address: a.cfg.GetString(cfgPprofAddress)} + pprofService := metrics.NewPprofService(a.log, pprofConfig) + a.services = append(a.services, pprofService) + go pprofService.Start() + + prometheusConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPrometheusEnabled), Address: a.cfg.GetString(cfgPrometheusAddress)} + prometheusService := metrics.NewPrometheusService(a.log, prometheusConfig) + a.services = append(a.services, prometheusService) + go prometheusService.Start() +} + +func (a *app) stopServices() { + ctx, cancel := context.WithTimeout(context.Background(), defaultShutdownTimeout) + defer cancel() + + for _, svc := range a.services { + svc.ShutDown(ctx) + } +} + +func (a *app) configureRouter(uploadRoutes *uploader.Uploader, downloadRoutes *downloader.Downloader) { r := router.New() r.RedirectTrailingSlash = true r.NotFound = func(r *fasthttp.RequestCtx) { @@ -274,55 +413,18 @@ func (a *app) Serve(ctx context.Context) { r.GET("/zip/{cid}/{prefix:*}", a.logger(downloadRoutes.DownloadZipped)) a.log.Info("added path /zip/{cid}/{prefix}") - pprofConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPprofEnabled), Address: a.cfg.GetString(cfgPprofAddress)} - pprof := metrics.NewPprofService(a.log, pprofConfig) - prometheusConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPrometheusEnabled), Address: a.cfg.GetString(cfgPrometheusAddress)} - prometheus := metrics.NewPrometheusService(a.log, prometheusConfig) - - bind := a.cfg.GetString(cfgListenAddress) - tlsCertPath := a.cfg.GetString(cfgTLSCertificate) - tlsKeyPath := a.cfg.GetString(cfgTLSKey) - a.webServer.Handler = r.Handler - - go pprof.Start() - go prometheus.Start() - - go func() { - var err error - if tlsCertPath == "" && tlsKeyPath == "" { - a.log.Info("running web server", zap.String("address", bind)) - err = a.webServer.ListenAndServe(bind) - } else { - a.log.Info("running web server (TLS-enabled)", zap.String("address", bind)) - err = a.webServer.ListenAndServeTLS(bind, tlsCertPath, tlsKeyPath) - } - if err != nil { - a.log.Fatal("could not start server", zap.Error(err)) - } - }() - - <-ctx.Done() - a.log.Info("shutting down web server", zap.Error(a.webServer.Shutdown())) - - ctx, cancel := context.WithTimeout(context.Background(), defaultShutdownTimeout) - defer cancel() - - pprof.ShutDown(ctx) - prometheus.ShutDown(ctx) - - close(a.webDone) } func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { - return fasthttp.RequestHandler(func(ctx *fasthttp.RequestCtx) { + return func(ctx *fasthttp.RequestCtx) { a.log.Info("request", zap.String("remote", ctx.RemoteAddr().String()), zap.ByteString("method", ctx.Method()), zap.ByteString("path", ctx.Path()), zap.ByteString("query", ctx.QueryArgs().QueryString()), zap.Uint64("id", ctx.ID())) h(ctx) - }) + } } func (a *app) AppParams() *utils.AppParams { diff --git a/integration_test.go b/integration_test.go index 9f55486..2f4cd12 100644 --- a/integration_test.go +++ b/integration_test.go @@ -82,8 +82,8 @@ func runServer() context.CancelFunc { cancelCtx, cancel := context.WithCancel(context.Background()) v := getDefaultConfig() - l := newLogger(v) - application := newApp(cancelCtx, WithConfig(v), WithLogger(l)) + l, lvl := newLogger(v) + application := newApp(cancelCtx, WithConfig(v), WithLogger(l, lvl)) go application.Serve(cancelCtx) return cancel diff --git a/main.go b/main.go index 5b4d188..f997955 100644 --- a/main.go +++ b/main.go @@ -2,65 +2,16 @@ package main import ( "context" - "fmt" "os/signal" "syscall" - - "github.com/spf13/viper" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" ) func main() { - var ( - v = settings() - l = newLogger(v) - ) - globalContext, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) - app := newApp(globalContext, WithLogger(l), WithConfig(v)) - go app.Serve(globalContext) - app.Wait() -} - -// newLogger constructs a zap.Logger instance for current application. -// Panics on failure. -// -// Logger is built from zap's production logging configuration with: -// - parameterized level (debug by default) -// - console encoding -// - ISO8601 time encoding -// -// Logger records a stack trace for all messages at or above fatal level. -// -// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. -func newLogger(v *viper.Viper) *zap.Logger { - var lvl zapcore.Level - lvlStr := v.GetString(cfgLoggerLevel) - err := lvl.UnmarshalText([]byte(lvlStr)) - if err != nil { - panic(fmt.Sprintf("incorrect logger level configuration %s (%v), "+ - "value should be one of %v", lvlStr, err, [...]zapcore.Level{ - zapcore.DebugLevel, - zapcore.InfoLevel, - zapcore.WarnLevel, - zapcore.ErrorLevel, - zapcore.DPanicLevel, - zapcore.PanicLevel, - zapcore.FatalLevel, - })) - } - - c := zap.NewProductionConfig() - c.Level = zap.NewAtomicLevelAt(lvl) - c.Encoding = "console" - c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - - l, err := c.Build( - zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), - ) - if err != nil { - panic(fmt.Sprintf("build zap logger instance: %v", err)) - } - - return l + globalContext, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + v := settings() + logger, atomicLevel := newLogger(v) + + application := newApp(globalContext, WithLogger(logger, atomicLevel), WithConfig(v)) + go application.Serve(globalContext) + application.Wait() } diff --git a/settings.go b/settings.go index c9e6648..e687141 100644 --- a/settings.go +++ b/settings.go @@ -13,6 +13,8 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/valyala/fasthttp" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) const ( @@ -108,7 +110,7 @@ func settings() *viper.Viper { flags.StringP(cmdWallet, "w", "", `path to the wallet`) flags.String(cmdAddress, "", `address of wallet account`) - config := flags.String(cmdConfig, "", "config path") + flags.String(cmdConfig, "", "config path") flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout") flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout") flags.Duration(cfgRebalance, defaultRebalanceTimer, "gRPC connection rebalance timer") @@ -213,9 +215,7 @@ func settings() *viper.Viper { } if v.IsSet(cmdConfig) { - if cfgFile, err := os.Open(*config); err != nil { - panic(err) - } else if err := v.ReadConfig(cfgFile); err != nil { + if err := readConfig(v); err != nil { panic(err) } } @@ -230,3 +230,67 @@ func settings() *viper.Viper { return v } + +func readConfig(v *viper.Viper) error { + cfgFileName := v.GetString(cmdConfig) + cfgFile, err := os.Open(cfgFileName) + if err != nil { + return err + } + if err = v.ReadConfig(cfgFile); err != nil { + return err + } + + return cfgFile.Close() +} + +// newLogger constructs a zap.Logger instance for current application. +// Panics on failure. +// +// Logger is built from zap's production logging configuration with: +// - parameterized level (debug by default) +// - console encoding +// - ISO8601 time encoding +// +// Logger records a stack trace for all messages at or above fatal level. +// +// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. +func newLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { + lvl, err := getLogLevel(v) + if err != nil { + panic(err) + } + + c := zap.NewProductionConfig() + c.Level = zap.NewAtomicLevelAt(lvl) + c.Encoding = "console" + c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + + l, err := c.Build( + zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), + ) + if err != nil { + panic(fmt.Sprintf("build zap logger instance: %v", err)) + } + + return l, c.Level +} + +func getLogLevel(v *viper.Viper) (zapcore.Level, error) { + var lvl zapcore.Level + lvlStr := v.GetString(cfgLoggerLevel) + err := lvl.UnmarshalText([]byte(lvlStr)) + if err != nil { + return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+ + "value should be one of %v", lvlStr, err, [...]zapcore.Level{ + zapcore.DebugLevel, + zapcore.InfoLevel, + zapcore.WarnLevel, + zapcore.ErrorLevel, + zapcore.DPanicLevel, + zapcore.PanicLevel, + zapcore.FatalLevel, + }) + } + return lvl, nil +} From 1e05d8a935933d95a567f59f2cfdcfe9149163bb Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 8 Sep 2022 19:00:22 +0300 Subject: [PATCH 311/548] [#200] Reload resolvers on SIGHUP Signed-off-by: Denis Kirillov --- app.go | 32 ++++++---- resolver/resolver.go | 135 +++++++++++++++++++++++++++++-------------- utils/util.go | 4 +- 3 files changed, 116 insertions(+), 55 deletions(-) diff --git a/app.go b/app.go index 6a86726..e7c5c09 100644 --- a/app.go +++ b/app.go @@ -158,6 +158,21 @@ func newApp(ctx context.Context, opt ...Option) App { a.log.Fatal("failed to dial pool", zap.Error(err)) } + a.initResolver() + a.initMetrics() + + return a +} + +func (a *app) initResolver() { + var err error + a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) + if err != nil { + a.log.Fatal("failed to create resolver", zap.Error(err)) + } +} + +func (a *app) getResolverConfig() ([]string, *resolver.Config) { resolveCfg := &resolver.Config{ NeoFS: resolver.NewNeoFSResolver(a.pool), RPCAddress: a.cfg.GetString(cfgRPCEndpoint), @@ -169,18 +184,11 @@ func newApp(ctx context.Context, opt ...Option) App { a.log.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint)) } - if len(order) != 0 { - a.resolver, err = resolver.NewResolver(order, resolveCfg) - if err != nil { - a.log.Fatal("failed to create resolver", zap.Error(err)) - } - } else { - a.log.Info("container resolver is disabled") + if len(order) == 0 { + a.log.Info("container resolver will be disabled because of resolvers 'resolver_order' is empty") } - a.initMetrics() - - return a + return order, resolveCfg } func (a *app) initMetrics() { @@ -365,6 +373,10 @@ func (a *app) configReload() { a.logLevel.SetLevel(lvl) } + if err := a.resolver.UpdateResolvers(a.getResolverConfig()); err != nil { + a.log.Warn("failed to update resolvers", zap.Error(err)) + } + a.stopServices() a.startServices() diff --git a/resolver/resolver.go b/resolver/resolver.go index 9ea9eb8..388697e 100644 --- a/resolver/resolver.go +++ b/resolver/resolver.go @@ -2,7 +2,9 @@ package resolver import ( "context" + "errors" "fmt" + "sync" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/ns" @@ -13,6 +15,9 @@ const ( DNSResolver = "dns" ) +// ErrNoResolvers returns when trying to resolve container without any resolver. +var ErrNoResolvers = errors.New("no resolvers") + // NeoFS represents virtual connection to the NeoFS network. type NeoFS interface { // SystemDNS reads system DNS network parameters of the NeoFS. @@ -28,66 +33,116 @@ type Config struct { } type ContainerResolver struct { - Name string - resolve func(context.Context, string) (*cid.ID, error) - - next *ContainerResolver + mu sync.RWMutex + resolvers []*Resolver } -func (r *ContainerResolver) SetResolveFunc(fn func(context.Context, string) (*cid.ID, error)) { +type Resolver struct { + Name string + resolve func(context.Context, string) (*cid.ID, error) +} + +func (r *Resolver) SetResolveFunc(fn func(context.Context, string) (*cid.ID, error)) { r.resolve = fn } -func (r *ContainerResolver) Resolve(ctx context.Context, name string) (*cid.ID, error) { - cnrID, err := r.resolve(ctx, name) - if err != nil { - if r.next != nil { - cnrID, inErr := r.next.Resolve(ctx, name) - if inErr != nil { - return nil, fmt.Errorf("%s; %w", err.Error(), inErr) - } - return cnrID, nil - } - return nil, err - } - return cnrID, nil +func (r *Resolver) Resolve(ctx context.Context, name string) (*cid.ID, error) { + return r.resolve(ctx, name) } -func NewResolver(order []string, cfg *Config) (*ContainerResolver, error) { - if len(order) == 0 { - return nil, fmt.Errorf("resolving order must not be empty") - } - - bucketResolver, err := newResolver(order[len(order)-1], cfg, nil) +func NewContainerResolver(resolverNames []string, cfg *Config) (*ContainerResolver, error) { + resolvers, err := createResolvers(resolverNames, cfg) if err != nil { return nil, err } - for i := len(order) - 2; i >= 0; i-- { - resolverName := order[i] - next := bucketResolver + return &ContainerResolver{ + resolvers: resolvers, + }, nil +} - bucketResolver, err = newResolver(resolverName, cfg, next) +func createResolvers(resolverNames []string, cfg *Config) ([]*Resolver, error) { + resolvers := make([]*Resolver, len(resolverNames)) + for i, name := range resolverNames { + cnrResolver, err := newResolver(name, cfg) if err != nil { return nil, err } + resolvers[i] = cnrResolver } - return bucketResolver, nil + return resolvers, nil } -func newResolver(name string, cfg *Config, next *ContainerResolver) (*ContainerResolver, error) { +func (r *ContainerResolver) Resolve(ctx context.Context, cnrName string) (*cid.ID, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + var err error + for _, resolver := range r.resolvers { + cnrID, resolverErr := resolver.Resolve(ctx, cnrName) + if resolverErr != nil { + resolverErr = fmt.Errorf("%s: %w", resolver.Name, resolverErr) + if err == nil { + err = resolverErr + } else { + err = fmt.Errorf("%s: %w", err.Error(), resolverErr) + } + continue + } + return cnrID, nil + } + + if err != nil { + return nil, err + } + + return nil, ErrNoResolvers +} + +func (r *ContainerResolver) UpdateResolvers(resolverNames []string, cfg *Config) error { + r.mu.Lock() + defer r.mu.Unlock() + + if r.equals(resolverNames) { + return nil + } + + resolvers, err := createResolvers(resolverNames, cfg) + if err != nil { + return err + } + + r.resolvers = resolvers + + return nil +} + +func (r *ContainerResolver) equals(resolverNames []string) bool { + if len(r.resolvers) != len(resolverNames) { + return false + } + + for i := 0; i < len(resolverNames); i++ { + if r.resolvers[i].Name != resolverNames[i] { + return false + } + } + return true +} + +func newResolver(name string, cfg *Config) (*Resolver, error) { switch name { case DNSResolver: - return NewDNSResolver(cfg.NeoFS, next) + return NewDNSResolver(cfg.NeoFS) case NNSResolver: - return NewNNSResolver(cfg.RPCAddress, next) + return NewNNSResolver(cfg.RPCAddress) default: return nil, fmt.Errorf("unknown resolver: %s", name) } } -func NewDNSResolver(neoFS NeoFS, next *ContainerResolver) (*ContainerResolver, error) { +func NewDNSResolver(neoFS NeoFS) (*Resolver, error) { if neoFS == nil { return nil, fmt.Errorf("pool must not be nil for DNS resolver") } @@ -108,15 +163,13 @@ func NewDNSResolver(neoFS NeoFS, next *ContainerResolver) (*ContainerResolver, e return &cnrID, nil } - return &ContainerResolver{ - Name: DNSResolver, - + return &Resolver{ + Name: DNSResolver, resolve: resolveFunc, - next: next, }, nil } -func NewNNSResolver(rpcAddress string, next *ContainerResolver) (*ContainerResolver, error) { +func NewNNSResolver(rpcAddress string) (*Resolver, error) { var nns ns.NNS if err := nns.Dial(rpcAddress); err != nil { @@ -131,10 +184,8 @@ func NewNNSResolver(rpcAddress string, next *ContainerResolver) (*ContainerResol return &cnrID, nil } - return &ContainerResolver{ - Name: NNSResolver, - + return &Resolver{ + Name: NNSResolver, resolve: resolveFunc, - next: next, }, nil } diff --git a/utils/util.go b/utils/util.go index 73068f1..8255528 100644 --- a/utils/util.go +++ b/utils/util.go @@ -13,9 +13,7 @@ func GetContainerID(ctx context.Context, containerID string, resolver *resolver. cnrID := new(cid.ID) err := cnrID.DecodeString(containerID) if err != nil { - if resolver != nil { - cnrID, err = resolver.Resolve(ctx, containerID) - } + cnrID, err = resolver.Resolve(ctx, containerID) } return cnrID, err } From ce84dc7068c0042c6dffd444cf307280234ac183 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 9 Sep 2022 09:33:31 +0300 Subject: [PATCH 312/548] [#200] Reload upload/download settings on SIGHUP Signed-off-by: Denis Kirillov --- app.go | 33 ++++++++++++++++++++++++++++----- downloader/download.go | 17 +++++++++++++---- uploader/upload.go | 41 +++++++++++++++++++++++++++-------------- 3 files changed, 68 insertions(+), 23 deletions(-) diff --git a/app.go b/app.go index e7c5c09..c180353 100644 --- a/app.go +++ b/app.go @@ -41,6 +41,12 @@ type ( resolver *resolver.ContainerResolver metrics *gateMetrics services []*metrics.Service + settings *appSettings + } + + appSettings struct { + Uploader *uploader.Settings + Downloader *downloader.Settings } // App is an interface for the main gateway function. @@ -158,12 +164,22 @@ func newApp(ctx context.Context, opt ...Option) App { a.log.Fatal("failed to dial pool", zap.Error(err)) } + a.initAppSettings() a.initResolver() a.initMetrics() return a } +func (a *app) initAppSettings() { + a.settings = &appSettings{ + Uploader: &uploader.Settings{}, + Downloader: &downloader.Settings{}, + } + + a.updateSettings() +} + func (a *app) initResolver() { var err error a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) @@ -309,10 +325,8 @@ func (a *app) setHealthStatus() { } func (a *app) Serve(ctx context.Context) { - edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) - uploadRoutes := uploader.New(ctx, a.AppParams(), edts) - downloadSettings := downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)} - downloadRoutes := downloader.New(ctx, a.AppParams(), downloadSettings) + uploadRoutes := uploader.New(ctx, a.AppParams(), a.settings.Uploader) + downloadRoutes := downloader.New(ctx, a.AppParams(), a.settings.Downloader) // Configure router. a.configureRouter(uploadRoutes, downloadRoutes) @@ -358,7 +372,7 @@ LOOP: } func (a *app) configReload() { - a.log.Info("SIGHUP config reload") + a.log.Info("SIGHUP config reload started") if !a.cfg.IsSet(cmdConfig) { a.log.Warn("failed to reload config because it's missed") return @@ -380,8 +394,17 @@ func (a *app) configReload() { a.stopServices() a.startServices() + a.updateSettings() + a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled)) a.setHealthStatus() + + a.log.Info("SIGHUP config reload completed") +} + +func (a *app) updateSettings() { + a.settings.Uploader.SetDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) + a.settings.Downloader.SetZipCompression(a.cfg.GetBool(cfgZipCompression)) } func (a *app) startServices() { diff --git a/downloader/download.go b/downloader/download.go index e34e3b7..09b0f4a 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -13,6 +13,7 @@ import ( "path" "strconv" "strings" + "sync/atomic" "time" "unicode" "unicode/utf8" @@ -255,15 +256,23 @@ type Downloader struct { log *zap.Logger pool *pool.Pool containerResolver *resolver.ContainerResolver - settings Settings + settings *Settings } type Settings struct { - ZipCompression bool + zipCompression atomic.Bool +} + +func (s *Settings) ZipCompression() bool { + return s.zipCompression.Load() +} + +func (s *Settings) SetZipCompression(val bool) { + s.zipCompression.Store(val) } // New creates an instance of Downloader using specified options. -func New(ctx context.Context, params *utils.AppParams, settings Settings) *Downloader { +func New(ctx context.Context, params *utils.AppParams, settings *Settings) *Downloader { return &Downloader{ appCtx: ctx, log: params.Logger, @@ -385,7 +394,7 @@ func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string func (d *Downloader) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { method := zip.Store - if d.settings.ZipCompression { + if d.settings.ZipCompression() { method = zip.Deflate } diff --git a/uploader/upload.go b/uploader/upload.go index 4c1172c..0784249 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "strconv" + "sync/atomic" "time" "github.com/nspcc-dev/neofs-http-gw/resolver" @@ -28,12 +29,12 @@ const ( // Uploader is an upload request handler. type Uploader struct { - appCtx context.Context - log *zap.Logger - pool *pool.Pool - ownerID *user.ID - enableDefaultTimestamp bool - containerResolver *resolver.ContainerResolver + appCtx context.Context + log *zap.Logger + pool *pool.Pool + ownerID *user.ID + settings *Settings + containerResolver *resolver.ContainerResolver } type epochDurations struct { @@ -42,16 +43,28 @@ type epochDurations struct { blockPerEpoch uint64 } +type Settings struct { + defaultTimestamp atomic.Bool +} + +func (s *Settings) DefaultTimestamp() bool { + return s.defaultTimestamp.Load() +} + +func (s *Settings) SetDefaultTimestamp(val bool) { + s.defaultTimestamp.Store(val) +} + // New creates a new Uploader using specified logger, connection pool and // other options. -func New(ctx context.Context, params *utils.AppParams, enableDefaultTimestamp bool) *Uploader { +func New(ctx context.Context, params *utils.AppParams, settings *Settings) *Uploader { return &Uploader{ - appCtx: ctx, - log: params.Logger, - pool: params.Pool, - ownerID: params.Owner, - enableDefaultTimestamp: enableDefaultTimestamp, - containerResolver: params.Resolver, + appCtx: ctx, + log: params.Logger, + pool: params.Pool, + ownerID: params.Owner, + settings: settings, + containerResolver: params.Resolver, } } @@ -130,7 +143,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { attributes = append(attributes, *filename) } // sets Timestamp attribute if it wasn't set from header and enabled by settings - if _, ok := filtered[object.AttributeTimestamp]; !ok && u.enableDefaultTimestamp { + if _, ok := filtered[object.AttributeTimestamp]; !ok && u.settings.DefaultTimestamp() { timestamp := object.NewAttribute() timestamp.SetKey(object.AttributeTimestamp) timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) From 82eba97505c7a4c360d11fb022395a9e5f124c81 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 9 Sep 2022 09:57:48 +0300 Subject: [PATCH 313/548] [#200] Unregister metrics on shutdown to fix test Signed-off-by: Denis Kirillov --- app.go | 12 ++++++++++++ downloader/download.go | 2 +- integration_test.go | 7 ++++--- metrics/metrics.go | 11 ++++++++++- uploader/upload.go | 2 +- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/app.go b/app.go index c180353..39dfa9f 100644 --- a/app.go +++ b/app.go @@ -67,6 +67,7 @@ type ( GateMetricsProvider interface { SetHealth(int32) + Unregister() } ) @@ -243,6 +244,16 @@ func (m *gateMetrics) SetHealth(status int32) { m.provider.SetHealth(status) } +func (m *gateMetrics) Shutdown() { + m.mu.Lock() + if m.enabled { + m.provider.SetHealth(0) + m.enabled = false + } + m.provider.Unregister() + m.mu.Unlock() +} + func remove(list []string, element string) []string { for i, item := range list { if item == element { @@ -366,6 +377,7 @@ LOOP: a.log.Info("shutting down web server", zap.Error(a.webServer.Shutdown())) + a.metrics.Shutdown() a.stopServices() close(a.webDone) diff --git a/downloader/download.go b/downloader/download.go index 09b0f4a..97ef42f 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -13,7 +13,6 @@ import ( "path" "strconv" "strings" - "sync/atomic" "time" "unicode" "unicode/utf8" @@ -28,6 +27,7 @@ import ( oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/valyala/fasthttp" + "go.uber.org/atomic" "go.uber.org/zap" ) diff --git a/integration_test.go b/integration_test.go index 2f4cd12..0da6f5e 100644 --- a/integration_test.go +++ b/integration_test.go @@ -61,7 +61,7 @@ func TestIntegration(t *testing.T) { ctx, cancel2 := context.WithCancel(rootCtx) aioContainer := createDockerContainer(ctx, t, aioImage+version) - cancel := runServer() + server, cancel := runServer() clientPool := getPool(ctx, t, key) CID, err := createContainer(ctx, t, clientPool, ownerID, version) require.NoError(t, err, version) @@ -72,13 +72,14 @@ func TestIntegration(t *testing.T) { t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, ownerID, CID, version) }) cancel() + server.Wait() err = aioContainer.Terminate(ctx) require.NoError(t, err) cancel2() } } -func runServer() context.CancelFunc { +func runServer() (App, context.CancelFunc) { cancelCtx, cancel := context.WithCancel(context.Background()) v := getDefaultConfig() @@ -86,7 +87,7 @@ func runServer() context.CancelFunc { application := newApp(cancelCtx, WithConfig(v), WithLogger(l, lvl)) go application.Serve(cancelCtx) - return cancel + return application, cancel } func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID, version string) { diff --git a/metrics/metrics.go b/metrics/metrics.go index 52b5e88..4520557 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -63,6 +63,11 @@ func NewGateMetrics(p *pool.Pool) *GateMetrics { } } +func (g *GateMetrics) Unregister() { + g.stateMetrics.unregister() + prometheus.Unregister(&g.poolMetricsCollector) +} + func newStateMetrics() *stateMetrics { return &stateMetrics{ healthCheck: prometheus.NewGauge(prometheus.GaugeOpts{ @@ -78,6 +83,10 @@ func (m stateMetrics) register() { prometheus.MustRegister(m.healthCheck) } +func (m stateMetrics) unregister() { + prometheus.Unregister(m.healthCheck) +} + func (m stateMetrics) SetHealth(s int32) { m.healthCheck.Set(float64(s)) } @@ -160,7 +169,7 @@ func (m *poolMetricsCollector) Collect(ch chan<- prometheus.Metric) { m.requestDuration.Collect(ch) } -func (m poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) { +func (m *poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) { m.overallErrors.Describe(descs) m.overallNodeErrors.Describe(descs) m.overallNodeRequests.Describe(descs) diff --git a/uploader/upload.go b/uploader/upload.go index 0784249..d2cabb4 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "strconv" - "sync/atomic" "time" "github.com/nspcc-dev/neofs-http-gw/resolver" @@ -19,6 +18,7 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/nspcc-dev/neofs-sdk-go/user" "github.com/valyala/fasthttp" + "go.uber.org/atomic" "go.uber.org/zap" ) From ad2c7ca67102d21c72406406ab7c4da548f80c3c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 9 Sep 2022 19:00:04 +0300 Subject: [PATCH 314/548] [#200] Reload certs on SIGHUP Signed-off-by: Denis Kirillov --- app.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 15 deletions(-) diff --git a/app.go b/app.go index 39dfa9f..9b34267 100644 --- a/app.go +++ b/app.go @@ -3,7 +3,10 @@ package main import ( "context" "crypto/ecdsa" + "crypto/tls" + "errors" "fmt" + "net" "os" "os/signal" "strconv" @@ -45,8 +48,9 @@ type ( } appSettings struct { - Uploader *uploader.Settings - Downloader *downloader.Settings + Uploader *uploader.Settings + Downloader *downloader.Settings + TLSProvider *certProvider } // App is an interface for the main gateway function. @@ -174,8 +178,9 @@ func newApp(ctx context.Context, opt ...Option) App { func (a *app) initAppSettings() { a.settings = &appSettings{ - Uploader: &uploader.Settings{}, - Downloader: &downloader.Settings{}, + Uploader: &uploader.Settings{}, + Downloader: &downloader.Settings{}, + TLSProvider: &certProvider{Enabled: a.cfg.IsSet(cfgTLSCertificate) || a.cfg.IsSet(cfgTLSKey)}, } a.updateSettings() @@ -335,6 +340,43 @@ func (a *app) setHealthStatus() { a.metrics.SetHealth(1) } +type certProvider struct { + Enabled bool + + mu sync.RWMutex + certPath string + keyPath string + cert *tls.Certificate +} + +func (p *certProvider) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + if !p.Enabled { + return nil, errors.New("cert provider: disabled") + } + + p.mu.RLock() + defer p.mu.RUnlock() + return p.cert, nil +} + +func (p *certProvider) UpdateCert(certPath, keyPath string) error { + if !p.Enabled { + return fmt.Errorf("tls disabled") + } + + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return fmt.Errorf("cannot load TLS key pair from certFile '%s' and keyFile '%s': %w", certPath, keyPath, err) + } + + p.mu.Lock() + p.certPath = certPath + p.keyPath = keyPath + p.cert = &cert + p.mu.Unlock() + return nil +} + func (a *app) Serve(ctx context.Context) { uploadRoutes := uploader.New(ctx, a.AppParams(), a.settings.Uploader) downloadRoutes := downloader.New(ctx, a.AppParams(), a.settings.Downloader) @@ -344,21 +386,34 @@ func (a *app) Serve(ctx context.Context) { a.startServices() - bind := a.cfg.GetString(cfgListenAddress) - tlsCertPath := a.cfg.GetString(cfgTLSCertificate) - tlsKeyPath := a.cfg.GetString(cfgTLSKey) - go func() { var err error - if tlsCertPath == "" && tlsKeyPath == "" { + defer func() { + if err != nil { + a.log.Fatal("could not start server", zap.Error(err)) + } + }() + + bind := a.cfg.GetString(cfgListenAddress) + + if a.settings.TLSProvider.Enabled { + if err = a.settings.TLSProvider.UpdateCert(a.cfg.GetString(cfgTLSCertificate), a.cfg.GetString(cfgTLSKey)); err != nil { + return + } + + var ln net.Listener + if ln, err = net.Listen("tcp4", bind); err != nil { + return + } + lnTLS := tls.NewListener(ln, &tls.Config{ + GetCertificate: a.settings.TLSProvider.GetCertificate, + }) + + a.log.Info("running web server (TLS-enabled)", zap.String("address", bind)) + err = a.webServer.Serve(lnTLS) + } else { a.log.Info("running web server", zap.String("address", bind)) err = a.webServer.ListenAndServe(bind) - } else { - a.log.Info("running web server (TLS-enabled)", zap.String("address", bind)) - err = a.webServer.ListenAndServeTLS(bind, tlsCertPath, tlsKeyPath) - } - if err != nil { - a.log.Fatal("could not start server", zap.Error(err)) } }() @@ -417,6 +472,10 @@ func (a *app) configReload() { func (a *app) updateSettings() { a.settings.Uploader.SetDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) a.settings.Downloader.SetZipCompression(a.cfg.GetBool(cfgZipCompression)) + + if err := a.settings.TLSProvider.UpdateCert(a.cfg.GetString(cfgTLSCertificate), a.cfg.GetString(cfgTLSKey)); err != nil { + a.log.Warn("failed to reload TLS certs", zap.Error(err)) + } } func (a *app) startServices() { From c73eed9634e303be7008d0630e527e7166f4f077 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 12 Sep 2022 16:50:18 +0300 Subject: [PATCH 315/548] [#200] Add context to net.Listener Signed-off-by: Denis Kirillov --- app.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app.go b/app.go index 9b34267..7cc84cc 100644 --- a/app.go +++ b/app.go @@ -401,8 +401,9 @@ func (a *app) Serve(ctx context.Context) { return } + var lnConf net.ListenConfig var ln net.Listener - if ln, err = net.Listen("tcp4", bind); err != nil { + if ln, err = lnConf.Listen(ctx, "tcp4", bind); err != nil { return } lnTLS := tls.NewListener(ln, &tls.Config{ From 3fa0e3caefffeabad383fc0d153b4b6b328abd88 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 16 Sep 2022 15:47:16 +0300 Subject: [PATCH 316/548] [#200] Update docs Signed-off-by: Denis Kirillov --- CHANGELOG.md | 3 +++ downloader/download.go | 1 + uploader/upload.go | 1 + 3 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28f7d25..954fdf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ This document outlines major changes between releases. ## [Unreleased] +### Added +- Config reloading on SIGHUP (#200) + ## [0.24.0] - 2022-09-14 ### Fixed diff --git a/downloader/download.go b/downloader/download.go index 97ef42f..bb48b51 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -259,6 +259,7 @@ type Downloader struct { settings *Settings } +// Settings stores reloading parameters, so it has to provide atomic getters and setters. type Settings struct { zipCompression atomic.Bool } diff --git a/uploader/upload.go b/uploader/upload.go index d2cabb4..2db7c34 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -43,6 +43,7 @@ type epochDurations struct { blockPerEpoch uint64 } +// Settings stores reloading parameters, so it has to provide atomic getters and setters. type Settings struct { defaultTimestamp atomic.Bool } From b567a08a851f3accc311d0caf50a072ef16deb9f Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 26 Sep 2022 16:02:04 +0300 Subject: [PATCH 317/548] [#206] Refactor neofs 'not found' errors Signed-off-by: Denis Kirillov --- downloader/download.go | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index bb48b51..6891ef7 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -22,6 +22,7 @@ import ( "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/bearer" + "github.com/nspcc-dev/neofs-sdk-go/client" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" @@ -37,8 +38,6 @@ type request struct { log *zap.Logger } -var errObjectNotFound = errors.New("object not found") - const attributeFilePath = "FilePath" func isValidToken(s string) bool { @@ -232,22 +231,14 @@ func (r *request) handleNeoFSErr(err error, start time.Time) { zap.Stringer("elapsed", time.Since(start)), zap.Error(err), ) - var ( - msg = fmt.Sprintf("could not receive object: %v", err) - code = fasthttp.StatusBadRequest - cause = err - ) - for unwrap := errors.Unwrap(err); unwrap != nil; unwrap = errors.Unwrap(cause) { - cause = unwrap + + if client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err) { + response.Error(r.RequestCtx, "Not Found", fasthttp.StatusNotFound) + return } - if strings.Contains(cause.Error(), "not found") || - strings.Contains(cause.Error(), "can't fetch container info") { - code = fasthttp.StatusNotFound - msg = errObjectNotFound.Error() - } - - response.Error(r.RequestCtx, msg, code) + msg := fmt.Sprintf("could not receive object: %v", err) + response.Error(r.RequestCtx, msg, fasthttp.StatusBadRequest) } // Downloader is a download request handler. From 6190e2bd7214a51b7e5cb05654a79b887fe0c8ce Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 30 Sep 2022 16:41:24 +0300 Subject: [PATCH 318/548] [#209] Add check dirty version to CI Signed-off-by: Denis Kirillov --- .github/workflows/builds.yml | 3 +++ go.mod | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index d1f6b16..e1ed2b8 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -35,6 +35,9 @@ jobs: - name: Build CLI run: make + - name: Check version + run: if [[ $(make version) == *"dirty"* ]]; then exit 1; fi + build_image: needs: build_cli name: Build Docker image diff --git a/go.mod b/go.mod index 48814d4..ac501eb 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/testcontainers/testcontainers-go v0.13.0 github.com/valyala/fasthttp v1.34.0 + go.uber.org/atomic v1.9.0 go.uber.org/zap v1.18.1 ) @@ -81,7 +82,6 @@ require ( github.com/urfave/cli v1.22.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.opencensus.io v0.23.0 // indirect - go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.7.0 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect From 45c036f5201d9dd222af282371288b668a4fd241 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 3 Oct 2022 09:44:13 +0300 Subject: [PATCH 319/548] [#208] Add SIGHUP docs Signed-off-by: Denis Kirillov --- docs/gate-configuration.md | 86 +++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 28 deletions(-) diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 4661aa5..7812fb3 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -8,6 +8,36 @@ There are some custom types used for brevity: * `duration` -- string consisting of a number and a suffix. Suffix examples include `s` (seconds), `m` (minutes), `ms` ( milliseconds). + +# Reload on SIGHUP + +Some config values can be reloaded on SIGHUP signal. +Such parameters have special mark in tables below. + +You can send SIGHUP signal to app using the following command: + +```shell +$ kill -s SIGHUP +``` + +Example: + +```shell +$ ./bin/neofs-http-gw --config config.yaml &> http.log & +[1] 998346 + +$ cat http.log +# ... +2022-10-03T09:37:25.826+0300 info neofs-http-gw/app.go:332 starting application {"app_name": "neofs-http-gw", "version": "v0.24.0"} +# ... + +$ kill -s SIGHUP 998346 + +$ cat http.log +# ... +2022-10-03T09:38:16.205+0300 info neofs-http-gw/app.go:470 SIGHUP config reload completed +``` + # Structure | Section | Description | @@ -41,17 +71,17 @@ rebalance_timer: 30s pool_error_threshold: 100 ``` -| Parameter | Type | Default value | Description | -|------------------------|------------|----------------|------------------------------------------------------------------------------------| -| `listen_address` | `string` | `0.0.0.0:8082` | The address that the gateway is listening on. | -| `tls_certificate` | `string` | | Path to the TLS certificate. | -| `tls_key` | `string` | | Path to the TLS key. | -| `rpc_endpoint` | `string` | | The address of the RPC host to which the gateway connects to resolve bucket names. | -| `resolve_order` | `[]string` | `[nns, dns]` | Order of bucket name resolvers to use. | -| `connect_timeout` | `duration` | `10s` | Timeout to connect to a node. | -| `request_timeout` | `duration` | `15s` | Timeout to check node health during rebalance. | -| `rebalance_timer` | `duration` | `60s` | Interval to check node health. | -| `pool_error_threshold` | `uint32` | `100` | The number of errors on connection after which node is considered as unhealthy. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|------------------------|------------|---------------|----------------|------------------------------------------------------------------------------------| +| `listen_address` | `string` | | `0.0.0.0:8082` | The address that the gateway is listening on. | +| `tls_certificate` | `string` | yes | | Path to the TLS certificate. | +| `tls_key` | `string` | yes | | Path to the TLS key. | +| `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. | +| `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. | +| `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. | +| `request_timeout` | `duration` | | `15s` | Timeout to check node health during rebalance. | +| `rebalance_timer` | `duration` | | `60s` | Interval to check node health. | +| `pool_error_threshold` | `uint32` | | `100` | The number of errors on connection after which node is considered as unhealthy. | # `wallet` section @@ -106,9 +136,9 @@ logger: level: debug ``` -| Parameter | Type | Default value | Description | -|-----------|----------|---------------|----------------------------------------------------------------------------------------------------| -| `level` | `string` | `debug` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|----------|---------------|---------------|----------------------------------------------------------------------------------------------------| +| `level` | `string` | yes | `debug` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | # `web` section @@ -140,9 +170,9 @@ upload_header: use_default_timestamp: false ``` -| Parameter | Type | Default value | Description | -|-------------------------|--------|---------------|-------------------------------------------------------------| -| `use_default_timestamp` | `bool` | `false` | Create timestamp for object if it isn't provided by header. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-------------------------|--------|---------------|---------------|-------------------------------------------------------------| +| `use_default_timestamp` | `bool` | yes | `false` | Create timestamp for object if it isn't provided by header. | # `zip` section @@ -152,9 +182,9 @@ zip: compression: false ``` -| Parameter | Type | Default value | Description | -|---------------|--------|---------------|--------------------------------------------------------------| -| `compression` | `bool` | `false` | Enable zip compression when download files by common prefix. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------|--------|---------------|---------------|--------------------------------------------------------------| +| `compression` | `bool` | yes | `false` | Enable zip compression when download files by common prefix. | # `pprof` section @@ -167,10 +197,10 @@ pprof: address: localhost:8083 ``` -| Parameter | Type | Default value | Description | -|-----------|----------|------------------|-----------------------------------------| -| `enabled` | `bool` | `false` | Flag to enable the service. | -| `address` | `string` | `localhost:8083` | Address that service listener binds to. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|----------|---------------|------------------|-----------------------------------------| +| `enabled` | `bool` | yes | `false` | Flag to enable the service. | +| `address` | `string` | yes | `localhost:8083` | Address that service listener binds to. | # `prometheus` section @@ -182,7 +212,7 @@ prometheus: address: localhost:8084 ``` -| Parameter | Type | Default value | Description | -|-----------|----------|------------------|-----------------------------------------| -| `enabled` | `bool` | `false` | Flag to enable the service. | -| `address` | `string` | `localhost:8084` | Address that service listener binds to. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|----------|---------------|------------------|-----------------------------------------| +| `enabled` | `bool` | yes | `false` | Flag to enable the service. | +| `address` | `string` | yes | `localhost:8084` | Address that service listener binds to. | From 5eb3c6b26d1f02ef852a4cd40293b407e4bc2fc1 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 10 Oct 2022 12:08:24 +0300 Subject: [PATCH 320/548] [#212] Update SDK to support interrupt pool dial Signed-off-by: Denis Kirillov --- go.mod | 24 ++++++++++++------------ go.sum | 53 ++++++++++++++++++++++++++++++++++------------------- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index ac501eb..e0bd29e 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.17 require ( github.com/fasthttp/router v1.4.1 - github.com/nspcc-dev/neo-go v0.99.1 - github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220827080658-9e17cdfc7647 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220829114550-ee92df32032e - github.com/prometheus/client_golang v1.11.0 + github.com/nspcc-dev/neo-go v0.99.2 + github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20221005093543-3a91383f24a9 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20221007102402-8c682641bfd2 + github.com/prometheus/client_golang v1.13.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 @@ -22,7 +22,7 @@ require ( github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/hcsshim v0.9.2 // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1 // indirect + github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/btcsuite/btcd v0.22.0-beta // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect @@ -56,7 +56,7 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect github.com/nspcc-dev/hrw v1.0.9 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220727202624-6c7a401f776a // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220809123759-3094d3e0c14b // indirect github.com/nspcc-dev/neofs-contract v0.15.3 // indirect github.com/nspcc-dev/neofs-crypto v0.4.0 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect @@ -68,8 +68,8 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.30.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect github.com/sirupsen/logrus v1.8.1 // indirect @@ -85,13 +85,13 @@ require ( go.uber.org/multierr v1.7.0 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect + golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect - google.golang.org/grpc v1.45.0 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/grpc v1.48.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 592a895..af718cc 100644 --- a/go.sum +++ b/go.sum @@ -115,8 +115,9 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1 h1:zFRi26YWd7NIorBXe8UkevRl0dIvk/AnXHWaAiZG+Yk= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= +github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -191,6 +192,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= @@ -374,6 +376,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -404,9 +407,11 @@ github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -507,8 +512,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -741,15 +747,15 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= github.com/nspcc-dev/neo-go v0.99.1-pre.0.20220714084516-54849ef3e58e/go.mod h1:/y5Sl8p3YheTygriBtCCMWKkDOek8HcvSo5ds2rJtKI= -github.com/nspcc-dev/neo-go v0.99.1 h1:/dlDVVCqlmazusIu+Emd4TlwxjIOpNNU5ob7bwUH6Qk= -github.com/nspcc-dev/neo-go v0.99.1/go.mod h1:wvKMQXlE/M3eqtN0KIi1MjkEVTvYIJRrOMnJtqEkCxs= +github.com/nspcc-dev/neo-go v0.99.2 h1:Fq79FI6BJkj/XkgWtrURSdXgXIeBHCgbKauBw3LOvZ4= +github.com/nspcc-dev/neo-go v0.99.2/go.mod h1:9P0yWqhZX7i/ChJ+zjtiStO1uPTolPFUM+L5oNznU8E= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220713145417-4f184498bc42/go.mod h1:QBE0I30F2kOAISNpT5oks82yF4wkkUq3SCfI3Hqgx/Y= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220727202624-6c7a401f776a h1:ND5lYt9/cybrbS6/DVX0398a4ilt4/xE/QtEaOzJSGY= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220727202624-6c7a401f776a/go.mod h1:QBE0I30F2kOAISNpT5oks82yF4wkkUq3SCfI3Hqgx/Y= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220809123759-3094d3e0c14b h1:J7QZNmnO84esVuPbBo88fwAG4XVnDjlSTiO1ewLNCkQ= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220809123759-3094d3e0c14b/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220827080658-9e17cdfc7647 h1:00JDCprdZG15n8z58EaFvJlmvZ4uESUii8MSzk869JM= -github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20220827080658-9e17cdfc7647/go.mod h1:NAaDfOnFWIbAFkTj7pNQ+cknVue0JbdeRT9QQaeBCEY= +github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20221005093543-3a91383f24a9 h1:c9ovp4KuPyIBx4dVG4bmkePlmuN0au4BBtFGXALWFBM= +github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20221005093543-3a91383f24a9/go.mod h1:DRIr0Ic1s+6QgdqmNFNLIqMqd7lNMJfYwkczlm1hDtM= github.com/nspcc-dev/neofs-contract v0.15.3 h1:7+NwyTtxFAnIevz0hR/XxQf6R2Ej2scjVR2bnnJnhBM= github.com/nspcc-dev/neofs-contract v0.15.3/go.mod h1:BXVZUZUJxrmmDETglXHI8+5DSgn84B9y5DoSWqEjYCs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= @@ -759,8 +765,8 @@ github.com/nspcc-dev/neofs-crypto v0.4.0 h1:5LlrUAM5O0k1+sH/sktBtrgfWtq1pgpDs09f github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220829114550-ee92df32032e h1:4bw+na7QGjEHnZeNGhW31+l/MQ6N5NZ1LxCbldY0Rgo= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20220829114550-ee92df32032e/go.mod h1:zANAq4ZCA1YFX+DSzP/ydBv/Krj5sdkK5orpTISmZR4= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20221007102402-8c682641bfd2 h1:iOuKuX54KQVFUR3yYPk8IGioD4dtwzoz3IAolavuqKE= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20221007102402-8c682641bfd2/go.mod h1:RLGptWbksCAODl8bK+2YD9EBpzDJrfJR/eM0lBK87qk= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= @@ -851,8 +857,10 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -867,8 +875,10 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -880,8 +890,9 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1199,6 +1210,7 @@ golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -1216,6 +1228,7 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1226,8 +1239,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1336,8 +1350,8 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1434,7 +1448,6 @@ golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -1547,8 +1560,9 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1562,8 +1576,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= From ce0afda10e470694d24b77ce5c2cf1977aac60c1 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 10 Oct 2022 12:15:46 +0300 Subject: [PATCH 321/548] [#212] Update CHANGELOG.md Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 954fdf5..242ee1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This document outlines major changes between releases. ### Added - Config reloading on SIGHUP (#200) +- Stop pool dial on SIGINT (#212) ## [0.24.0] - 2022-09-14 From e4f8ab35c658747f027fea0b0534ef95435e6d8f Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 11 Oct 2022 16:15:40 +0300 Subject: [PATCH 322/548] [#212] Add new aio versions to integration tests Signed-off-by: Denis Kirillov --- integration_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration_test.go b/integration_test.go index 0da6f5e..22ea802 100644 --- a/integration_test.go +++ b/integration_test.go @@ -49,6 +49,8 @@ func TestIntegration(t *testing.T) { "0.27.5", "0.28.1", "0.29.0", + "0.30.0", + "0.32.0", "latest", } key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") From d34f106342aa0ec626ee287a132bab90989954d0 Mon Sep 17 00:00:00 2001 From: anastasia prasolova Date: Sun, 16 Oct 2022 20:35:59 +0300 Subject: [PATCH 323/548] Add CODEOWNERS file Signed-off-by: anastasia prasolova --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..1414545 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @alexvanin @masterSplinter01 @KirillovDenis From 1b172e581123db9cb9fc77fe24f1789b20a4c14a Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Sun, 16 Oct 2022 20:34:34 +0400 Subject: [PATCH 324/548] [#213] Makefile: Add help Signed-off-by: Angira Kekteeva --- Makefile | 3 +++ help.mk | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 help.mk diff --git a/Makefile b/Makefile index d961a94..005fa28 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ dep: @CGO_ENABLED=0 \ go mod tidy -v && echo OK +# Run `make %` in Golang container, for more information run `make help.docker/%` docker/%: $(if $(filter $*,all $(BINS)), \ @echo "=> Running 'make $*' in clean Docker environment" && \ @@ -109,3 +110,5 @@ version: clean: rm -rf vendor rm -rf $(BINDIR) + +include help.mk diff --git a/help.mk b/help.mk new file mode 100644 index 0000000..b5fbbc9 --- /dev/null +++ b/help.mk @@ -0,0 +1,22 @@ +.PHONY: help + +# Show this help prompt +help: + @echo ' Usage:' + @echo '' + @echo ' make ' + @echo '' + @echo ' Targets:' + @echo '' + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9.%_/-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq + +# Show help for docker/% IGNORE +help.docker/%: + $(eval TARGETS:=$(notdir all lint) ${BINS}) + @echo ' Usage:' + @echo '' + @echo ' make docker/% -- Run `make %` in Golang container' + @echo '' + @echo ' Supported docker targets:' + @echo '' + @$(foreach bin, $(TARGETS), echo ' ' $(bin);) From c258db6ab334a692a9d9a4699efd71b5aef0b155 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 27 Oct 2022 16:32:55 +0300 Subject: [PATCH 325/548] [#218] Unify system attribute format for GET/HEAD Signed-off-by: Denis Kirillov --- downloader/head.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/downloader/head.go b/downloader/head.go index eafed21..3267228 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -4,6 +4,7 @@ import ( "io" "net/http" "strconv" + "strings" "time" "github.com/nspcc-dev/neofs-http-gw/response" @@ -55,6 +56,9 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { if !isValidToken(key) || !isValidValue(val) { continue } + if strings.HasPrefix(key, utils.SystemAttributePrefix) { + key = systemBackwardTranslator(key) + } r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) switch key { case object.AttributeTimestamp: From a5ebc2ac6678f7a0b91ab350802d7a9b6668f6e9 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 26 Oct 2022 13:54:25 +0300 Subject: [PATCH 326/548] Release v0.25.0 Signed-off-by: Alex Vanin --- CHANGELOG.md | 13 +++++++++++-- VERSION | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 242ee1e..3c0b71f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,17 @@ This document outlines major changes between releases. ## [Unreleased] +## [0.25.0] - 2022-10-31 + ### Added -- Config reloading on SIGHUP (#200) +- Config reloading on SIGHUP (#200, #208) - Stop pool dial on SIGINT (#212) +- Makefile help (#213) + +### Changed +- Update NeoFS error handling (#206) +- GitHub actions updates (#205, #209) +- Unified system attribute format for GET and HEAD (#213) ## [0.24.0] - 2022-09-14 @@ -227,4 +235,5 @@ releases. [0.22.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.21.0...v0.22.0 [0.23.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.22.0...v0.23.0 [0.24.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.23.0...v0.24.0 -[Unreleased]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.24.0...master +[0.25.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.24.0...v0.25.0 +[Unreleased]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.0...master diff --git a/VERSION b/VERSION index 6897c00..5bd9725 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.24.0 +v0.25.0 From 131f2dbcfc1d247ce2430c3cdb6eee84eca1d0bf Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 2 Nov 2022 13:10:45 +0300 Subject: [PATCH 327/548] [#225] Run CI on support branch PR Signed-off-by: Alex Vanin --- .github/workflows/builds.yml | 1 + .github/workflows/codeql-analysis.yml | 4 ++-- .github/workflows/dco.yml | 1 + .github/workflows/tests.yml | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index e1ed2b8..64e2e5e 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - master + - 'support/*' types: [opened, synchronize] paths-ignore: - '**/*.md' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5a107f1..8598516 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,10 +13,10 @@ name: "CodeQL" on: push: - branches: [ master ] + branches: [ master, 'support/*' ] pull_request: # The branches below must be a subset of the branches above - branches: [ master ] + branches: [ master, 'support/*' ] schedule: - cron: '35 8 * * 1' diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index 40ed8fc..397e961 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - master + - 'support/*' jobs: commits_check_job: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5c8b409..cf1d462 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - master + - 'support/*' types: [opened, synchronize] paths-ignore: - '**/*.md' From b2db6300c4f6ec486d4ea540e91d3ea566669fc7 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 2 Nov 2022 15:08:14 +0300 Subject: [PATCH 328/548] [#222] Fix zip streaming Skip objects with invalid FilePath Signed-off-by: Denis Kirillov --- downloader/download.go | 50 +++++++++++++++++++++++++++--------------- integration_test.go | 6 ++--- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 6891ef7..296f58d 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -23,6 +23,7 @@ import ( "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/bearer" "github.com/nspcc-dev/neofs-sdk-go/client" + "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" @@ -38,8 +39,6 @@ type request struct { log *zap.Logger } -const attributeFilePath = "FilePath" - func isValidToken(s string) bool { for _, c := range s { if c <= ' ' || c > 127 { @@ -384,14 +383,26 @@ func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string return d.pool.SearchObjects(d.appCtx, prm) } +func (d *Downloader) getContainer(cnrID cid.ID) (container.Container, error) { + var prm pool.PrmContainerGet + prm.SetContainerID(cnrID) + + return d.pool.GetContainer(d.appCtx, prm) +} + func (d *Downloader) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { method := zip.Store if d.settings.ZipCompression() { method = zip.Deflate } + filePath := getZipFilePath(obj) + if len(filePath) == 0 || filePath[len(filePath)-1] == '/' { + return nil, fmt.Errorf("invalid filepath '%s'", filePath) + } + return zw.CreateHeader(&zip.FileHeader{ - Name: getZipFilePath(obj), + Name: filePath, Method: method, Modified: time.Now(), }) @@ -416,7 +427,20 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { return } - resSearch, err := d.search(c, containerID, attributeFilePath, prefix, object.MatchCommonPrefix) + // check if container exists here to be able to return 404 error, + // otherwise we get this error only in object iteration step + // and client get 200 OK. + if _, err = d.getContainer(*containerID); err != nil { + log.Error("could not check container existence", zap.Error(err)) + if client.IsErrContainerNotFound(err) { + response.Error(c, "Not Found", fasthttp.StatusNotFound) + return + } + response.Error(c, "could not check container existence: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + resSearch, err := d.search(c, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error("could not search for objects", zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) @@ -450,29 +474,19 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { addr.SetObject(id) if err = d.zipObject(zipWriter, addr, btoken, bufZip); err != nil { - return true + log.Error("failed to add object to archive", zap.String("oid", id.EncodeToString()), zap.Error(err)) } return false }) if errIter != nil { log.Error("iterating over selected objects failed", zap.Error(errIter)) - response.Error(c, "iterating over selected objects: "+errIter.Error(), fasthttp.StatusBadRequest) - return } else if !called { log.Error("objects not found") - response.Error(c, "objects not found", fasthttp.StatusNotFound) - return } - if err == nil { - err = zipWriter.Close() - } - - if err != nil { - log.Error("file streaming failure", zap.Error(err)) - response.Error(c, "file streaming failure: "+err.Error(), fasthttp.StatusInternalServerError) - return + if err = zipWriter.Close(); err != nil { + log.Error("close zip writer", zap.Error(err)) } }) } @@ -511,7 +525,7 @@ func (d *Downloader) zipObject(zipWriter *zip.Writer, addr oid.Address, btoken * func getZipFilePath(obj *object.Object) string { for _, attr := range obj.Attributes() { - if attr.Key() == attributeFilePath { + if attr.Key() == object.AttributeFilePath { return attr.Value() } } diff --git a/integration_test.go b/integration_test.go index 22ea802..b8b1e57 100644 --- a/integration_test.go +++ b/integration_test.go @@ -28,8 +28,6 @@ import ( "github.com/testcontainers/testcontainers-go/wait" ) -const attributeFilePath = "FilePath" - type putResponse struct { CID string `json:"container_id"` OID string `json:"object_id"` @@ -249,8 +247,8 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"} contents := []string{"content of file1", "content of file2"} - attributes1 := map[string]string{attributeFilePath: names[0]} - attributes2 := map[string]string{attributeFilePath: names[1]} + attributes1 := map[string]string{object.AttributeFilePath: names[0]} + attributes2 := map[string]string{object.AttributeFilePath: names[1]} putObject(ctx, t, clientPool, ownerID, CID, contents[0], attributes1) putObject(ctx, t, clientPool, ownerID, CID, contents[1], attributes2) From 9dad47502e2429d0f43574aacf5a263d0a070fdb Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 7 Nov 2022 15:31:37 +0300 Subject: [PATCH 329/548] [#222] Update docs Signed-off-by: Denis Kirillov --- CHANGELOG.md | 3 +++ README.md | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c0b71f..9fb890f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ This document outlines major changes between releases. ## [Unreleased] +### Fixed +- Download zip archive when `FilePath` is invalid (#222) + ## [0.25.0] - 2022-10-31 ### Added diff --git a/README.md b/README.md index 8d03a43..1db3023 100644 --- a/README.md +++ b/README.md @@ -347,8 +347,8 @@ You can download some dir (files with the same prefix) in zip (it will be compre $ wget http://localhost:8082/zip/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/common/prefix ``` -**Note:** the objects must have a `FilePath` attribute, otherwise they will not be in the zip archive. -You can upload file with this attribute using `curl`: +**Note:** the objects must have a valid `FilePath` attribute (it should not contain trailing `/`), +otherwise they will not be in the zip archive. You can upload file with this attribute using `curl`: ``` $ curl -F 'file=@cat.jpeg;filename=cat.jpeg' -H 'X-Attribute-FilePath: common/prefix/cat.jpeg' http://localhost:8082/upload/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ From be47263c422e3dc15d54d29ec39191419af29985 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 8 Nov 2022 09:55:13 +0300 Subject: [PATCH 330/548] [#214] Support the Date header on upload Signed-off-by: Denis Kirillov --- uploader/filter.go | 4 +--- uploader/filter_test.go | 2 +- uploader/upload.go | 13 ++++++++++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/uploader/filter.go b/uploader/filter.go index e9e9395..e453cfe 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -70,7 +70,7 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]str return result } -func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations) error { +func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations, now time.Time) error { expirationInEpoch := headers[object.SysAttributeExpEpoch] if timeRFC3339, ok := headers[utils.ExpirationRFC3339Attr]; ok { @@ -79,7 +79,6 @@ func prepareExpirationHeader(headers map[string]string, epochDurations *epochDur return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, utils.ExpirationRFC3339Attr) } - now := time.Now().UTC() if expTime.Before(now) { return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, utils.ExpirationRFC3339Attr) } @@ -94,7 +93,6 @@ func prepareExpirationHeader(headers map[string]string, epochDurations *epochDur } expTime := time.Unix(value, 0) - now := time.Now() if expTime.Before(now) { return fmt.Errorf("value %s of header %s must be in the future", timestamp, utils.ExpirationTimestampAttr) } diff --git a/uploader/filter_test.go b/uploader/filter_test.go index f1e300d..585fac8 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -178,7 +178,7 @@ func TestPrepareExpirationHeader(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - err := prepareExpirationHeader(tc.headers, tc.durations) + err := prepareExpirationHeader(tc.headers, tc.durations, time.Now()) if tc.err { require.Error(t, err) } else { diff --git a/uploader/upload.go b/uploader/upload.go index 2db7c34..8e65456 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "net/http" "strconv" "time" @@ -121,7 +122,17 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { response.Error(c, "could not get epoch durations from network info: "+err.Error(), fasthttp.StatusBadRequest) return } - if err = prepareExpirationHeader(filtered, epochDuration); err != nil { + + now := time.Now() + if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { + if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { + log.Warn("could not parse client time", zap.String("Date header", string(rawHeader)), zap.Error(err)) + } else { + now = parsed + } + } + + if err = prepareExpirationHeader(filtered, epochDuration, now); err != nil { log.Error("could not parse expiration header", zap.Error(err)) response.Error(c, "could not parse expiration header: "+err.Error(), fasthttp.StatusBadRequest) return From 9fda0397e4aec7d0dc09885a63b589241e4f363a Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 8 Nov 2022 09:56:01 +0300 Subject: [PATCH 331/548] [#214] Update CHANGELOG.md Signed-off-by: Denis Kirillov --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fb890f..af685f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ This document outlines major changes between releases. ### Fixed - Download zip archive when `FilePath` is invalid (#222) +### Added +- Support the `Date` header on upload (#214) + ## [0.25.0] - 2022-10-31 ### Added From d2ced3ccbba9c3bf8b1ed778776180bb573db12a Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 26 Oct 2022 17:20:25 +0300 Subject: [PATCH 332/548] [#216] Clarify url encoding in README.md Signed-off-by: Denis Kirillov --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1db3023..5ffc87a 100644 --- a/README.md +++ b/README.md @@ -305,8 +305,9 @@ where `$ATTRIBUTE_NAME` is the name of the attribute we want to use, `$ATTRIBUTE_VALUE` is the value of this attribute that the target object should have. -**NB!** The attribute key and value must be url encoded, i.e., if you want to download an object with the attribute value -`a cat`, the value in the request must be `a+cat`. In the same way with the attribute key. +**NB!** The attribute key and value should be url encoded, i.e., if you want to download an object with the attribute value +`a cat`, the value in the request must be `a+cat`. In the same way with the attribute key. If you don't escape such values +everything can still work (for example you can use `d@ta` without encoding) but it's HIGHLY RECOMMENDED to encode all your attributes. If multiple objects have specified attribute with specified value, then the first one of them is returned (and you can't get others via this interface). From ef0c17372fd67d30fcd1a2a84c5644e26b423354 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 27 Oct 2022 14:18:48 +0300 Subject: [PATCH 333/548] [#216] Add routes specification Signed-off-by: Denis Kirillov --- README.md | 2 + docs/api.md | 309 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 311 insertions(+) create mode 100644 docs/api.md diff --git a/README.md b/README.md index 5ffc87a..8baf533 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ NeoFS HTTP Gateway bridges NeoFS internal protocol and HTTP standard. - you can download one file per request from the NeoFS Network - you can upload one file per request into the NeoFS Network +See available routes in [specification](./docs/api.md). + ## Installation ```go install github.com/nspcc-dev/neofs-http-gw``` diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..c7dd48b --- /dev/null +++ b/docs/api.md @@ -0,0 +1,309 @@ +# HTTP Gateway Specification + +| Route | Description | +|-------------------------------------------------|----------------------------------------------| +| `/upload/{cid}` | [Put object](#put-object) | +| `/get/{cid}/{oid}` | [Get object](#get-object) | +| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) | +| `/zip/{cid}/{prefix}` | [Download objects in archive](#download-zip) | + +**Note:** `cid` parameter can be base58 encoded container ID or container name +(the name must be registered in NNS, see appropriate section in [README](../README.md#nns)). + +Route parameters can be: + +* `Single` - match a single path segment (cannot contain `/` and be empty) +* `Catch-All` - match everything (such parameter usually the last one in routes) +* `Query` - regular query parameter + +### Bearer token + +All routes can accept [bearer token](../README.md#authentication) from: + +* `Authorization` header with `Bearer` type and base64-encoded token in + credentials field +* `Bearer` cookie with base64-encoded token contents + +Example: + +Header: + +``` +Authorization: Bearer ChA5Gev0d8JI26tAtWyyQA3WEhsKGTVxfQ56a0uQeFmOO63mqykBS1HNpw1rxSgaBgiyEBjODyIhAyxcn89Bj5fwCfXlj5HjSYjonHSErZoXiSqeyh0ZQSb2MgQIARAB +``` + +Cookie: + +``` +cookie: Bearer=ChA5Gev0d8JI26tAtWyyQA3WEhsKGTVxfQ56a0uQeFmOO63mqykBS1HNpw1rxSgaBgiyEBjODyIhAyxcn89Bj5fwCfXlj5HjSYjonHSErZoXiSqeyh0ZQSb2MgQIARAB +``` + +## Put object + +Route: `/upload/{cid}` + +| Route parameter | Type | Description | +|-----------------|--------|---------------------------------------------------------| +| `cid` | Single | Base58 encoded container ID or container name from NNS. | + +### Methods + +#### POST + +Upload file as object with attributes to NeoFS. + +##### Request + +###### Headers + +| Header | Description | +|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------| +| Common headers | See [bearer token](#bearer-token). | +| `X-Attribute-Neofs-*` | Used to set system NeoFS object attributes
(e.g. use "X-Attribute-Neofs-Expiration-Epoch" to set `__NEOFS__EXPIRATION_EPOCH` attribute). | +| `X-Attribute-*` | Used to set regular object attributes
(e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | +| `Date` | This header is used to calculate the right `__NEOFS__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | + +There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority): + +1. `X-Attribute-Neofs-Expiration-Epoch: 100` +2. `X-Attribute-Neofs-Expiration-Duration: 24h30m` +3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797` +4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z` + +which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way. + +If you don't specify the `X-Attribute-Timestamp` header the `Timestamp` attribute can be set anyway +(see http-gw [configuration](gate-configuration.md#upload-header-section)). + +The `X-Attribute-*` headers must be unique. If you provide several the same headers only one will be used. +Attribute key and value must be valid utf8 string. All attributes in sum must not be greater than 3mb. + +###### Body + +Body must contain multipart form with file. +The `filename` field from the multipart form will be set as `FileName` attribute of object +(can be overriden by `X-Attribute-FileName` header). + +##### Response + +###### Status codes + +| Status | Description | +|--------|----------------------------------------------| +| 200 | Object created successfully. | +| 400 | Some error occurred during object uploading. | + +## Get object + +Route: `/get/{cid}/{oid}?[download=true]` + +| Route parameter | Type | Description | +|-----------------|--------|------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `cid` | Single | Base58 encoded container ID or container name from NNS. | +| `oid` | Single | Base58 encoded object ID. | +| `download` | Query | Set the `Content-Disposition` header as `attachment` in response.
This make the browser to download object as file instead of showing it on the page. | + +### Methods + +#### GET + +Get an object (payload and attributes) by an address. + +##### Request + +###### Headers + +| Header | Description | +|----------------|------------------------------------| +| Common headers | See [bearer token](#bearer-token). | + +##### Response + +###### Headers + +| Header | Description | +|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Disposition` | Indicate how to browsers should treat file.
Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | + +###### Status codes + +| Status | Description | +|--------|------------------------------------------------| +| 200 | Object got successfully. | +| 400 | Some error occurred during object downloading. | +| 404 | Container or object not found. | + +#### HEAD + +Get an object attributes by an address. + +##### Request + +###### Headers + +| Header | Description | +|----------------|------------------------------------| +| Common headers | See [bearer token](#bearer-token). | + +##### Response + +###### Headers + +| Header | Description | +|-----------------------|--------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | + +###### Status codes + +| Status | Description | +|--------|---------------------------------------------------| +| 200 | Object head successfully. | +| 400 | Some error occurred during object HEAD operation. | +| 404 | Container or object not found. | + +## Search object + +Route: `/get_by_attribute/{cid}/{attr_key}/{attr_val}?[download=true]` + +| Route parameter | Type | Description | +|-----------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------| +| `cid` | Single | Base58 encoded container ID or container name from NNS. | +| `attr_key` | Single | Object attribute key to search. | +| `attr_val` | Catch-All | Object attribute value to match. | +| `download` | Query | Set the `Content-Disposition` header as `attachment` in response. This make the browser to download object as file instead of showing it on the page. | + +### Methods + +#### GET + +Find and get an object (payload and attributes) by a specific attribute. +If more than one object is found, an arbitrary one will be returned. + +##### Request + +###### Headers + +| Header | Description | +|----------------|------------------------------------| +| Common headers | See [bearer token](#bearer-token). | + +##### Response + +###### Headers + +| Header | Description | +|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Disposition` | Indicate how to browsers should treat file.
Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | + +###### Status codes + +| Status | Description | +|--------|------------------------------------------------| +| 200 | Object got successfully. | +| 400 | Some error occurred during object downloading. | +| 404 | Container or object not found. | + +#### HEAD + +Get object attributes by a specific attribute. +If more than one object is found, an arbitrary one will be used to get attributes. + +##### Request + +###### Headers + +| Header | Description | +|----------------|------------------------------------| +| Common headers | See [bearer token](#bearer-token). | + +##### Response + +###### Headers + +| Header | Description | +|-----------------------|--------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | + +###### Status codes + +| Status | Description | +|--------|---------------------------------------| +| 200 | Object head successfully. | +| 400 | Some error occurred during operation. | +| 404 | Container or object not found. | + +## Download zip + +Route: `/zip/{cid}/{prefix}` + +| Route parameter | Type | Description | +|-----------------|-----------|---------------------------------------------------------| +| `cid` | Single | Base58 encoded container ID or container name from NNS. | +| `prefix` | Catch-All | Prefix for object attribute `FilePath` to match. | + +### Methods + +#### GET + +Find objects by prefix for `FilePath` attributes. Return found objects in zip archive. +Name of files in archive sets to `FilePath` attribute of objects. +Time of files sets to time when object has started downloading. +You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` route. + +Archive can be compressed (see http-gw [configuration](gate-configuration.md#zip-section)). + +##### Request + +###### Headers + +| Header | Description | +|----------------|------------------------------------| +| Common headers | See [bearer token](#bearer-token). | + +##### Response + +###### Headers + +| Header | Description | +|-----------------------|-------------------------------------------------------------------------------------------------------------------| +| `Content-Disposition` | Indicate how to browsers should treat file (`attachment`). Set `filename` as `archive.zip`. | +| `Content-Type` | Indicate content type of object. Set to `application/zip` | + +###### Status codes + +| Status | Description | +|--------|-----------------------------------------------------| +| 200 | Object got successfully. | +| 400 | Some error occurred during object downloading. | +| 404 | Container or objects not found. | +| 500 | Some inner error (e.g. error on streaming objects). | From 4a0188b8f973ef6beadd49ab45237e2186d82058 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 14 Nov 2022 10:47:40 +0300 Subject: [PATCH 334/548] [#215] Mention caching strategy in README.md Signed-off-by: Denis Kirillov --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 8baf533..206d735 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,11 @@ set of reply headers generated using the following rules: if they can be safely represented in HTTP header), for example `FileName` attribute becomes `X-Attribute-FileName` header +##### Caching strategy + +HTTP Gateway doesn't control caching (doesn't anything with the `Cache-Control` header). Caching strategy strictly +depends on application use case. So it should be carefully done by proxy server. + ### Uploading You can POST files to `/upload/$CID` path where `$CID` is a container ID or its name if NNS is enabled. The From a6ec194c2af780250acbf2e5b4fdf8f0fde8c840 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 8 Nov 2022 15:56:36 +0300 Subject: [PATCH 335/548] [#233] Update sdk Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + go.mod | 18 +++++++++--------- go.sum | 37 ++++++++++++++++++++++--------------- resolver/resolver.go | 6 +++++- 4 files changed, 37 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af685f6..fdb0867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This document outlines major changes between releases. ### Fixed - Download zip archive when `FilePath` is invalid (#222) +- Only one peer must be healthy to init pool (#233) ### Added - Support the `Date` header on upload (#214) diff --git a/go.mod b/go.mod index e0bd29e..d76778f 100644 --- a/go.mod +++ b/go.mod @@ -4,17 +4,17 @@ go 1.17 require ( github.com/fasthttp/router v1.4.1 - github.com/nspcc-dev/neo-go v0.99.2 - github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20221005093543-3a91383f24a9 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20221007102402-8c682641bfd2 + github.com/nspcc-dev/neo-go v0.99.4 + github.com/nspcc-dev/neofs-api-go/v2 v2.14.0 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221114144617-d047289182a1 github.com/prometheus/client_golang v1.13.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.0 github.com/testcontainers/testcontainers-go v0.13.0 github.com/valyala/fasthttp v1.34.0 - go.uber.org/atomic v1.9.0 - go.uber.org/zap v1.18.1 + go.uber.org/atomic v1.10.0 + go.uber.org/zap v1.23.0 ) require ( @@ -56,8 +56,8 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect github.com/nspcc-dev/hrw v1.0.9 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220809123759-3094d3e0c14b // indirect - github.com/nspcc-dev/neofs-contract v0.15.3 // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262 // indirect + github.com/nspcc-dev/neofs-contract v0.16.0 // indirect github.com/nspcc-dev/neofs-crypto v0.4.0 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/nspcc-dev/tzhash v1.6.1 // indirect @@ -82,7 +82,7 @@ require ( github.com/urfave/cli v1.22.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.opencensus.io v0.23.0 // indirect - go.uber.org/multierr v1.7.0 // indirect + go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect diff --git a/go.sum b/go.sum index af718cc..991aa8e 100644 --- a/go.sum +++ b/go.sum @@ -739,6 +739,7 @@ github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxx github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/dbft v0.0.0-20220629112714-fd49ca59d354/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= +github.com/nspcc-dev/dbft v0.0.0-20220902113116-58a5e763e647/go.mod h1:g9xisXmX9NP9MjioaTe862n9SlZTrP+6PVUWLBYOr98= github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 h1:n4ZaFCKt1pQJd7PXoMJabZWK9ejjbLOVrkl/lOUmshg= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= @@ -746,18 +747,18 @@ github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= -github.com/nspcc-dev/neo-go v0.99.1-pre.0.20220714084516-54849ef3e58e/go.mod h1:/y5Sl8p3YheTygriBtCCMWKkDOek8HcvSo5ds2rJtKI= -github.com/nspcc-dev/neo-go v0.99.2 h1:Fq79FI6BJkj/XkgWtrURSdXgXIeBHCgbKauBw3LOvZ4= github.com/nspcc-dev/neo-go v0.99.2/go.mod h1:9P0yWqhZX7i/ChJ+zjtiStO1uPTolPFUM+L5oNznU8E= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220713145417-4f184498bc42/go.mod h1:QBE0I30F2kOAISNpT5oks82yF4wkkUq3SCfI3Hqgx/Y= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220809123759-3094d3e0c14b h1:J7QZNmnO84esVuPbBo88fwAG4XVnDjlSTiO1ewLNCkQ= +github.com/nspcc-dev/neo-go v0.99.4 h1:8Y+SdRxksC72a4PNkcGCh/aaQinh9Gu+c5LilbcsXOI= +github.com/nspcc-dev/neo-go v0.99.4/go.mod h1:mKTolfRUfKjFso5HPvGSQtUZc70n0VKBMs16eGuC5gA= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220809123759-3094d3e0c14b/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262 h1:UTmSLZw5OpD/JPE1B5Vf98GF0zu2/Hsqq1lGLtStTUE= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20221005093543-3a91383f24a9 h1:c9ovp4KuPyIBx4dVG4bmkePlmuN0au4BBtFGXALWFBM= -github.com/nspcc-dev/neofs-api-go/v2 v2.13.2-0.20221005093543-3a91383f24a9/go.mod h1:DRIr0Ic1s+6QgdqmNFNLIqMqd7lNMJfYwkczlm1hDtM= -github.com/nspcc-dev/neofs-contract v0.15.3 h1:7+NwyTtxFAnIevz0hR/XxQf6R2Ej2scjVR2bnnJnhBM= -github.com/nspcc-dev/neofs-contract v0.15.3/go.mod h1:BXVZUZUJxrmmDETglXHI8+5DSgn84B9y5DoSWqEjYCs= +github.com/nspcc-dev/neofs-api-go/v2 v2.14.0 h1:jhuN8Ldqz7WApvUJRFY0bjRXE1R3iCkboMX5QVZhHVk= +github.com/nspcc-dev/neofs-api-go/v2 v2.14.0/go.mod h1:DRIr0Ic1s+6QgdqmNFNLIqMqd7lNMJfYwkczlm1hDtM= +github.com/nspcc-dev/neofs-contract v0.16.0 h1:/K5IMwZOlUCooxpe//73RW20HTfHvwRL088gppU5DM8= +github.com/nspcc-dev/neofs-contract v0.16.0/go.mod h1:gN5bo2TlMvLbySImmg76DVj3jVmYgti2VVlQ+h/tcr0= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= @@ -765,8 +766,8 @@ github.com/nspcc-dev/neofs-crypto v0.4.0 h1:5LlrUAM5O0k1+sH/sktBtrgfWtq1pgpDs09f github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20221007102402-8c682641bfd2 h1:iOuKuX54KQVFUR3yYPk8IGioD4dtwzoz3IAolavuqKE= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6.0.20221007102402-8c682641bfd2/go.mod h1:RLGptWbksCAODl8bK+2YD9EBpzDJrfJR/eM0lBK87qk= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221114144617-d047289182a1 h1:SAB+pu0aXXxOsBxBNZG9F5c1YLCpSAW5HhBI5Id43d8= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221114144617-d047289182a1/go.mod h1:7bH8zabbewpXstaXKoV0Tk3lV7KFYadi5edLU5yDERY= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= @@ -963,14 +964,17 @@ github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1075,19 +1079,22 @@ go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= -go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= +go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/resolver/resolver.go b/resolver/resolver.go index 388697e..6fdf948 100644 --- a/resolver/resolver.go +++ b/resolver/resolver.go @@ -6,6 +6,7 @@ import ( "fmt" "sync" + "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/ns" ) @@ -177,7 +178,10 @@ func NewNNSResolver(rpcAddress string) (*Resolver, error) { } resolveFunc := func(_ context.Context, name string) (*cid.ID, error) { - cnrID, err := nns.ResolveContainerName(name) + var d container.Domain + d.SetName(name) + + cnrID, err := nns.ResolveContainerDomain(d) if err != nil { return nil, fmt.Errorf("couldn't resolve container '%s': %w", name, err) } From 408d914347e5665b1057baea00d1ed33aa55c27d Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 15 Nov 2022 18:06:53 +0300 Subject: [PATCH 336/548] [#234] Update SDK to support timeout for stream Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + app.go | 1 + config/config.env | 2 ++ config/config.yaml | 1 + docs/gate-configuration.md | 2 ++ go.mod | 2 +- go.sum | 4 ++-- settings.go | 3 +++ 8 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdb0867..3b682d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This document outlines major changes between releases. ### Added - Support the `Date` header on upload (#214) +- Timeout for individual operations in streaming RPC (#234) ## [0.25.0] - 2022-10-31 diff --git a/app.go b/app.go index 7cc84cc..8e72555 100644 --- a/app.go +++ b/app.go @@ -137,6 +137,7 @@ func newApp(ctx context.Context, opt ...Option) App { var prm pool.InitParameters prm.SetKey(key) prm.SetNodeDialTimeout(a.cfg.GetDuration(cfgConTimeout)) + prm.SetNodeStreamTimeout(a.cfg.GetDuration(cfgStreamTimeout)) prm.SetHealthcheckTimeout(a.cfg.GetDuration(cfgReqTimeout)) prm.SetClientRebalanceInterval(a.cfg.GetDuration(cfgRebalance)) prm.SetErrorThreshold(a.cfg.GetUint32(cfgPoolErrorThreshold)) diff --git a/config/config.env b/config/config.env index d6258fb..b66dae0 100644 --- a/config/config.env +++ b/config/config.env @@ -79,6 +79,8 @@ HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=false # Timeout to dial node. HTTP_GW_CONNECT_TIMEOUT=5s +# Timeout for individual operations in streaming RPC. +HTTP_GW_STREAM_TIMEOUT=10s # Timeout to check node health during rebalance. HTTP_GW_REQUEST_TIMEOUT=5s # Interval to check nodes health. diff --git a/config/config.yaml b/config/config.yaml index ef784ed..38a9ff0 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -82,6 +82,7 @@ upload_header: use_default_timestamp: false # Create timestamp for object if it isn't provided by header. connect_timeout: 5s # Timeout to dial node. +stream_timeout: 10s # Timeout for individual operations in streaming RPC. request_timeout: 5s # Timeout to check node health during rebalance. rebalance_timer: 30s # Interval to check nodes health. pool_error_threshold: 100 # The number of errors on connection after which node is considered as unhealthy. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 7812fb3..2a69a1a 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -66,6 +66,7 @@ resolve_order: - dns connect_timeout: 5s +stream_timeout: 10s request_timeout: 5s rebalance_timer: 30s pool_error_threshold: 100 @@ -79,6 +80,7 @@ pool_error_threshold: 100 | `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. | | `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. | | `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. | +| `stream_timeout` | `duration` | | `10s` | Timeout for individual operations in streaming RPC. | | `request_timeout` | `duration` | | `15s` | Timeout to check node health during rebalance. | | `rebalance_timer` | `duration` | | `60s` | Interval to check node health. | | `pool_error_threshold` | `uint32` | | `100` | The number of errors on connection after which node is considered as unhealthy. | diff --git a/go.mod b/go.mod index d76778f..87d1dc4 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.99.4 github.com/nspcc-dev/neofs-api-go/v2 v2.14.0 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221114144617-d047289182a1 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221115140820-b4b07a3c4e11 github.com/prometheus/client_golang v1.13.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 diff --git a/go.sum b/go.sum index 991aa8e..a4e3530 100644 --- a/go.sum +++ b/go.sum @@ -766,8 +766,8 @@ github.com/nspcc-dev/neofs-crypto v0.4.0 h1:5LlrUAM5O0k1+sH/sktBtrgfWtq1pgpDs09f github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221114144617-d047289182a1 h1:SAB+pu0aXXxOsBxBNZG9F5c1YLCpSAW5HhBI5Id43d8= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221114144617-d047289182a1/go.mod h1:7bH8zabbewpXstaXKoV0Tk3lV7KFYadi5edLU5yDERY= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221115140820-b4b07a3c4e11 h1:dcofGaVR6najaIdh0JOXoxqV1MLJWnJ3pPMhD5gowV4= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221115140820-b4b07a3c4e11/go.mod h1:7bH8zabbewpXstaXKoV0Tk3lV7KFYadi5edLU5yDERY= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/settings.go b/settings.go index e687141..b042ab5 100644 --- a/settings.go +++ b/settings.go @@ -21,6 +21,7 @@ const ( defaultRebalanceTimer = 60 * time.Second defaultRequestTimeout = 15 * time.Second defaultConnectTimeout = 10 * time.Second + defaultStreamTimeout = 10 * time.Second defaultShutdownTimeout = 15 * time.Second @@ -46,6 +47,7 @@ const ( // Pool config. cfgConTimeout = "connect_timeout" + cfgStreamTimeout = "stream_timeout" cfgReqTimeout = "request_timeout" cfgRebalance = "rebalance_timer" cfgPoolErrorThreshold = "pool_error_threshold" @@ -112,6 +114,7 @@ func settings() *viper.Viper { flags.String(cmdAddress, "", `address of wallet account`) flags.String(cmdConfig, "", "config path") flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout") + flags.Duration(cfgStreamTimeout, defaultStreamTimeout, "gRPC individual message timeout") flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout") flags.Duration(cfgRebalance, defaultRebalanceTimer, "gRPC connection rebalance timer") From 7b1410968e66f7d692eec2bcc977eb4a6ca0204b Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Thu, 17 Nov 2022 13:17:06 +0400 Subject: [PATCH 337/548] [#236] Fix config example Signed-off-by: Angira Kekteeva --- config/config.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.env b/config/config.env index b66dae0..c92306a 100644 --- a/config/config.env +++ b/config/config.env @@ -86,7 +86,7 @@ HTTP_GW_REQUEST_TIMEOUT=5s # Interval to check nodes health. HTTP_GW_REBALANCE_TIMER=30s # The number of errors on connection after which node is considered as unhealthy -S3_GW_POOL_ERROR_THRESHOLD=100 +HTTP_GW_POOL_ERROR_THRESHOLD=100 # Enable zip compression to download files by common prefix. HTTP_GW_ZIP_COMPRESSION=false From f3d58e4ef00de8e9a0eeda408fd25e02b21570e8 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Thu, 24 Nov 2022 17:49:21 +0300 Subject: [PATCH 338/548] [#228] add support of multiple sockets Signed-off-by: Pavel Pogodaev --- app.go | 132 +++++++++++++++++--------------------------- config/config.env | 14 +++-- config/config.yaml | 14 ++++- integration_test.go | 2 +- server.go | 124 +++++++++++++++++++++++++++++++++++++++++ settings.go | 64 ++++++++++++++++----- 6 files changed, 247 insertions(+), 103 deletions(-) create mode 100644 server.go diff --git a/app.go b/app.go index 8e72555..6eb0945 100644 --- a/app.go +++ b/app.go @@ -3,10 +3,8 @@ package main import ( "context" "crypto/ecdsa" - "crypto/tls" - "errors" "fmt" - "net" + "net/http" "os" "os/signal" "strconv" @@ -45,12 +43,12 @@ type ( metrics *gateMetrics services []*metrics.Service settings *appSettings + servers []Server } appSettings struct { - Uploader *uploader.Settings - Downloader *downloader.Settings - TLSProvider *certProvider + Uploader *uploader.Settings + Downloader *downloader.Settings } // App is an interface for the main gateway function. @@ -179,9 +177,8 @@ func newApp(ctx context.Context, opt ...Option) App { func (a *app) initAppSettings() { a.settings = &appSettings{ - Uploader: &uploader.Settings{}, - Downloader: &downloader.Settings{}, - TLSProvider: &certProvider{Enabled: a.cfg.IsSet(cfgTLSCertificate) || a.cfg.IsSet(cfgTLSKey)}, + Uploader: &uploader.Settings{}, + Downloader: &downloader.Settings{}, } a.updateSettings() @@ -341,43 +338,6 @@ func (a *app) setHealthStatus() { a.metrics.SetHealth(1) } -type certProvider struct { - Enabled bool - - mu sync.RWMutex - certPath string - keyPath string - cert *tls.Certificate -} - -func (p *certProvider) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { - if !p.Enabled { - return nil, errors.New("cert provider: disabled") - } - - p.mu.RLock() - defer p.mu.RUnlock() - return p.cert, nil -} - -func (p *certProvider) UpdateCert(certPath, keyPath string) error { - if !p.Enabled { - return fmt.Errorf("tls disabled") - } - - cert, err := tls.LoadX509KeyPair(certPath, keyPath) - if err != nil { - return fmt.Errorf("cannot load TLS key pair from certFile '%s' and keyFile '%s': %w", certPath, keyPath, err) - } - - p.mu.Lock() - p.certPath = certPath - p.keyPath = keyPath - p.cert = &cert - p.mu.Unlock() - return nil -} - func (a *app) Serve(ctx context.Context) { uploadRoutes := uploader.New(ctx, a.AppParams(), a.settings.Uploader) downloadRoutes := downloader.New(ctx, a.AppParams(), a.settings.Downloader) @@ -386,38 +346,16 @@ func (a *app) Serve(ctx context.Context) { a.configureRouter(uploadRoutes, downloadRoutes) a.startServices() + a.initServers(ctx) - go func() { - var err error - defer func() { - if err != nil { - a.log.Fatal("could not start server", zap.Error(err)) + for i := range a.servers { + go func(i int) { + a.log.Info("starting server", zap.String("address", a.servers[i].Address())) + if err := a.webServer.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed { + a.log.Fatal("listen and serve", zap.Error(err)) } - }() - - bind := a.cfg.GetString(cfgListenAddress) - - if a.settings.TLSProvider.Enabled { - if err = a.settings.TLSProvider.UpdateCert(a.cfg.GetString(cfgTLSCertificate), a.cfg.GetString(cfgTLSKey)); err != nil { - return - } - - var lnConf net.ListenConfig - var ln net.Listener - if ln, err = lnConf.Listen(ctx, "tcp4", bind); err != nil { - return - } - lnTLS := tls.NewListener(ln, &tls.Config{ - GetCertificate: a.settings.TLSProvider.GetCertificate, - }) - - a.log.Info("running web server (TLS-enabled)", zap.String("address", bind)) - err = a.webServer.Serve(lnTLS) - } else { - a.log.Info("running web server", zap.String("address", bind)) - err = a.webServer.ListenAndServe(bind) - } - }() + }(i) + } sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGHUP) @@ -460,6 +398,10 @@ func (a *app) configReload() { a.log.Warn("failed to update resolvers", zap.Error(err)) } + if err := a.updateServers(); err != nil { + a.log.Warn("failed to reload server parameters", zap.Error(err)) + } + a.stopServices() a.startServices() @@ -474,10 +416,6 @@ func (a *app) configReload() { func (a *app) updateSettings() { a.settings.Uploader.SetDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) a.settings.Downloader.SetZipCompression(a.cfg.GetBool(cfgZipCompression)) - - if err := a.settings.TLSProvider.UpdateCert(a.cfg.GetString(cfgTLSCertificate), a.cfg.GetString(cfgTLSKey)); err != nil { - a.log.Warn("failed to reload TLS certs", zap.Error(err)) - } } func (a *app) startServices() { @@ -543,3 +481,37 @@ func (a *app) AppParams() *utils.AppParams { Resolver: a.resolver, } } + +func (a *app) initServers(ctx context.Context) { + serversInfo := fetchServers(a.cfg) + + a.servers = make([]Server, len(serversInfo)) + for i, serverInfo := range serversInfo { + a.log.Info("added server", + zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled), + zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile)) + a.servers[i] = newServer(ctx, serverInfo, a.log) + } +} + +func (a *app) updateServers() error { + serversInfo := fetchServers(a.cfg) + + if len(serversInfo) != len(a.servers) { + return fmt.Errorf("invalid servers configuration: length mismatch: old '%d', new '%d", len(a.servers), len(serversInfo)) + } + + for i, serverInfo := range serversInfo { + if serverInfo.Address != a.servers[i].Address() { + return fmt.Errorf("invalid servers configuration: addresses mismatch: old '%s', new '%s", a.servers[i].Address(), serverInfo.Address) + } + + if serverInfo.TLS.Enabled { + if err := a.servers[i].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { + return fmt.Errorf("failed to update tls certs: %w", err) + } + } + } + + return nil +} diff --git a/config/config.env b/config/config.env index c92306a..fbf7327 100644 --- a/config/config.env +++ b/config/config.env @@ -17,12 +17,14 @@ HTTP_GW_PROMETHEUS_ADDRESS=localhost:8084 # Log level. HTTP_GW_LOGGER_LEVEL=debug -# Address to bind. -HTTP_GW_LISTEN_ADDRESS=0.0.0.0:443 -# Provide cert to enable TLS. -HTTP_GW_TLS_CERTIFICATE=/path/to/tls/cert -# Provide key to enable TLS. -HTTP_GW_TLS_KEY=/path/to/tls/key +HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:443 +HTTP_GW_SERVER_0_TLS_ENABLED=false +HTTP_GW_SERVER_0_TLS_CERT_FILE=/path/to/tls/cert +HTTP_GW_SERVER_0_TLS_KEY_FILE=/path/to/tls/key +HTTP_GW_SERVER_1_ADDRESS=0.0.0.0:444 +HTTP_GW_SERVER_1_TLS_ENABLED=true +HTTP_GW_SERVER_1_TLS_CERT_FILE=/path/to/tls/cert +HTTP_GW_SERVER_1_TLS_KEY_FILE=/path/to/tls/key # Nodes configuration. # This configuration make the gateway use the first node (grpc://s01.neofs.devenv:8080) diff --git a/config/config.yaml b/config/config.yaml index 38a9ff0..fc2fada 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -13,9 +13,17 @@ prometheus: logger: level: debug # Log level. -listen_address: 0.0.0.0:443 # Address to bind. -tls_certificate: /path/to/tls/cert # Provide cert to enable TLS. -tls_key: /path/to/tls/key # Provide key to enable TLS. +server: + - address: 0.0.0.0:8080 + tls: + enabled: false + cert_file: /path/to/cert + key_file: /path/to/key + - address: 0.0.0.0:8081 + tls: + enabled: false + cert_file: /path/to/cert + key_file: /path/to/key # Nodes configuration. # This configuration make the gateway use the first node (grpc://s01.neofs.devenv:8080) diff --git a/integration_test.go b/integration_test.go index b8b1e57..4607ef6 100644 --- a/integration_test.go +++ b/integration_test.go @@ -335,7 +335,7 @@ func getDefaultConfig() *viper.Viper { v.SetDefault(cfgPeers+".0.priority", 1) v.SetDefault(cfgRPCEndpoint, "http://localhost:30333") - v.SetDefault(cfgListenAddress, testListenAddress) + v.SetDefault("server.0.address", testListenAddress) return v } diff --git a/server.go b/server.go new file mode 100644 index 0000000..3843081 --- /dev/null +++ b/server.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "sync" + + "go.uber.org/zap" +) + +type ( + ServerInfo struct { + Address string + TLS ServerTLSInfo + } + + ServerTLSInfo struct { + Enabled bool + CertFile string + KeyFile string + } + + Server interface { + Address() string + Listener() net.Listener + UpdateCert(certFile, keyFile string) error + } + + server struct { + address string + listener net.Listener + tlsProvider *certProvider + } + + certProvider struct { + Enabled bool + + mu sync.RWMutex + certPath string + keyPath string + cert *tls.Certificate + } +) + +func (s *server) Address() string { + return s.address +} + +func (s *server) Listener() net.Listener { + return s.listener +} + +func (s *server) UpdateCert(certFile, keyFile string) error { + return s.tlsProvider.UpdateCert(certFile, keyFile) +} + +func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) *server { + var lic net.ListenConfig + ln, err := lic.Listen(ctx, "tcp", serverInfo.Address) + if err != nil { + logger.Fatal("could not prepare listener", zap.String("address", serverInfo.Address), zap.Error(err)) + } + + tlsProvider := &certProvider{ + Enabled: serverInfo.TLS.Enabled, + } + + if serverInfo.TLS.Enabled { + if err = tlsProvider.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { + logger.Fatal("failed to update cert", zap.Error(err)) + } + + ln = tls.NewListener(ln, &tls.Config{ + GetCertificate: tlsProvider.GetCertificate, + }) + } + + return &server{ + address: serverInfo.Address, + listener: ln, + tlsProvider: tlsProvider, + } +} + +func (p *certProvider) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + if !p.Enabled { + return nil, errors.New("cert provider: disabled") + } + + p.mu.RLock() + defer p.mu.RUnlock() + return p.cert, nil +} + +func (p *certProvider) UpdateCert(certPath, keyPath string) error { + if !p.Enabled { + return fmt.Errorf("tls disabled") + } + + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return fmt.Errorf("cannot load TLS key pair from certFile '%s' and keyFile '%s': %w", certPath, keyPath, err) + } + + p.mu.Lock() + p.certPath = certPath + p.keyPath = keyPath + p.cert = &cert + p.mu.Unlock() + return nil +} + +func (p *certProvider) FilePaths() (string, string) { + if !p.Enabled { + return "", "" + } + + p.mu.RLock() + defer p.mu.RUnlock() + return p.certPath, p.keyPath +} diff --git a/settings.go b/settings.go index b042ab5..bb841d0 100644 --- a/settings.go +++ b/settings.go @@ -27,9 +27,10 @@ const ( defaultPoolErrorThreshold uint32 = 100 - cfgListenAddress = "listen_address" - cfgTLSCertificate = "tls_certificate" - cfgTLSKey = "tls_key" + cfgServer = "server" + cfgTLSEnabled = "tls.enabled" + cfgTLSCertFile = "tls.cert_file" + cfgTLSKeyFile = "tls.key_file" // Web. cfgWebReadBufferSize = "web.read_buffer_size" @@ -76,13 +77,14 @@ const ( cfgZipCompression = "zip.compression" // Command line args. - cmdHelp = "help" - cmdVersion = "version" - cmdPprof = "pprof" - cmdMetrics = "metrics" - cmdWallet = "wallet" - cmdAddress = "address" - cmdConfig = "config" + cmdHelp = "help" + cmdVersion = "version" + cmdPprof = "pprof" + cmdMetrics = "metrics" + cmdWallet = "wallet" + cmdAddress = "address" + cmdConfig = "config" + cmdListenAddress = "listen_address" ) var ignore = map[string]struct{}{ @@ -118,9 +120,9 @@ func settings() *viper.Viper { flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout") flags.Duration(cfgRebalance, defaultRebalanceTimer, "gRPC connection rebalance timer") - flags.String(cfgListenAddress, "0.0.0.0:8082", "address to listen") - flags.String(cfgTLSCertificate, "", "TLS certificate path") - flags.String(cfgTLSKey, "", "TLS key path") + flags.String(cmdListenAddress, "0.0.0.0:8080", "addresses to listen") + flags.String(cfgTLSCertFile, "", "TLS certificate path") + flags.String(cfgTLSKeyFile, "", "TLS key path") peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order") @@ -171,10 +173,24 @@ func settings() *viper.Viper { panic(err) } + if err := v.BindPFlag(cfgServer+".0.address", flags.Lookup(cmdListenAddress)); err != nil { + panic(err) + } + if err := v.BindPFlag(cfgServer+".0."+cfgTLSKeyFile, flags.Lookup(cfgTLSKeyFile)); err != nil { + panic(err) + } + if err := v.BindPFlag(cfgServer+".0."+cfgTLSCertFile, flags.Lookup(cfgTLSCertFile)); err != nil { + panic(err) + } + if err := flags.Parse(os.Args); err != nil { panic(err) } + if v.IsSet(cfgServer+".0."+cfgTLSKeyFile) && v.IsSet(cfgServer+".0."+cfgTLSCertFile) { + v.Set(cfgServer+".0."+cfgTLSEnabled, true) + } + if resolveMethods != nil { v.SetDefault(cfgResolveOrder, *resolveMethods) } @@ -297,3 +313,25 @@ func getLogLevel(v *viper.Viper) (zapcore.Level, error) { } return lvl, nil } + +func fetchServers(v *viper.Viper) []ServerInfo { + var servers []ServerInfo + + for i := 0; ; i++ { + key := cfgServer + "." + strconv.Itoa(i) + "." + + var serverInfo ServerInfo + serverInfo.Address = v.GetString(key + "address") + serverInfo.TLS.Enabled = v.GetBool(key + cfgTLSEnabled) + serverInfo.TLS.KeyFile = v.GetString(key + cfgTLSKeyFile) + serverInfo.TLS.CertFile = v.GetString(key + cfgTLSCertFile) + + if serverInfo.Address == "" { + break + } + + servers = append(servers, serverInfo) + } + + return servers +} From 8f1d84ba8d73317f722a4e678216eab15f8bc71a Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Thu, 24 Nov 2022 17:49:35 +0300 Subject: [PATCH 339/548] [#228] update docs Signed-off-by: Pavel Pogodaev --- CHANGELOG.md | 16 ++++++++++++++++ docs/gate-configuration.md | 34 +++++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b682d6..088df83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ This document outlines major changes between releases. ## [Unreleased] +### Added +- Multiple server listeners (#228) + +### Updating from v0.25.0 +Make sure your configuration is valid: + +If you configure application using environment variables change: +* `HTTP_GW_LISTEN_ADDRESS` -> `HTTP_GW_SERVER_0_ADDRESS` +* `HTTP_GW_TLS_CERT_FILE` -> `HTTP_GW_SERVER_0_TLS_CERT_FILE` (and set `HTTP_GW_SERVER_0_TLS_ENABLED=true`) +* `HTTP_GW_TLS_KEY_FILE` -> `HTTP_GW_SERVER_0_TLS_KEY_FILE` (and set `HTTP_GW_SERVER_0_TLS_ENABLED=true`) + +If you configure application using `.yaml` file change: +* `listen_address` -> `server.0.address` +* `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`) +* `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`) + ### Fixed - Download zip archive when `FilePath` is invalid (#222) - Only one peer must be healthy to init pool (#233) diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 2a69a1a..5944e39 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -47,6 +47,7 @@ $ cat http.log | `peers` | [Nodes configuration](#peers-section) | | `logger` | [Logger configuration](#logger-section) | | `web` | [Web configuration](#web-section) | +| `server` | [Server configuration](#server-section) | | `upload-header` | [Upload header configuration](#upload-header-section) | | `zip` | [ZIP configuration](#zip-section) | | `pprof` | [Pprof configuration](#pprof-section) | @@ -56,10 +57,6 @@ $ cat http.log # General section ```yaml -listen_address: 0.0.0.0:8082 -tls_certificate: /path/to/tls/cert -tls_key: /path/to/tls/key - rpc_endpoint: http://morph-chain.neofs.devenv:30333 resolve_order: - nns @@ -74,9 +71,6 @@ pool_error_threshold: 100 | Parameter | Type | SIGHUP reload | Default value | Description | |------------------------|------------|---------------|----------------|------------------------------------------------------------------------------------| -| `listen_address` | `string` | | `0.0.0.0:8082` | The address that the gateway is listening on. | -| `tls_certificate` | `string` | yes | | Path to the TLS certificate. | -| `tls_key` | `string` | yes | | Path to the TLS key. | | `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. | | `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. | | `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. | @@ -131,6 +125,32 @@ peers: | `priority` | `int` | `1` | It allows to group nodes and don't switch group until all nodes with the same priority will be unhealthy. The lower the value, the higher the priority. | | `weight` | `float` | `1` | Weight of node in the group with the same priority. Distribute requests to nodes proportionally to these values. | +# `server` section + +You can specify several listeners for server. For example, for `http` and `https`. + +```yaml +server: + - address: 0.0.0.0:8080 + tls: + enabled: false + cert_file: /path/to/cert + key_file: /path/to/key + - address: 0.0.0.0:8081 + tls: + enabled: true + cert_file: /path/to/another/cert + key_file: /path/to/another/key +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------------|----------|---------------|----------------|-----------------------------------------------| +| `address` | `string` | | `0.0.0.0:8080` | The address that the gateway is listening on. | +| `tls.enabled` | `bool` | | false | Enable TLS or not. | +| `tls.cert_file` | `string` | yes | | Path to the TLS certificate. | +| `tls.key_file` | `string` | yes | | Path to the key. | + + # `logger` section ```yaml From 278376643a46db50d6e04cc73d853f30fc1f3708 Mon Sep 17 00:00:00 2001 From: Artem Tataurov Date: Thu, 17 Nov 2022 18:01:38 +0300 Subject: [PATCH 340/548] [#221] add error response on key duplicates Signed-off-by: Artem Tataurov --- CHANGELOG.md | 1 + integration_test.go | 38 ++++++++++++++++++++++++++++++++++++++ uploader/filter.go | 22 +++++++++++++++------- uploader/filter_test.go | 22 +++++++++++++++++++++- uploader/upload.go | 7 ++++++- 5 files changed, 81 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 088df83..596457d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ If you configure application using `.yaml` file change: ### Added - Support the `Date` header on upload (#214) +- Add error response on attribute duplicates (#221) - Timeout for individual operations in streaming RPC (#234) ## [0.25.0] - 2022-10-31 diff --git a/integration_test.go b/integration_test.go index 4607ef6..d7cb8d0 100644 --- a/integration_test.go +++ b/integration_test.go @@ -67,6 +67,7 @@ func TestIntegration(t *testing.T) { require.NoError(t, err, version) t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID, version) }) + t.Run("put with duplicate keys "+version, func(t *testing.T) { putWithDuplicateKeys(t, CID) }) t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, ownerID, CID, version) }) t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID, version) }) t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, ownerID, CID, version) }) @@ -171,6 +172,43 @@ func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnr } } +func putWithDuplicateKeys(t *testing.T, CID cid.ID) { + url := testHost + "/upload/" + CID.String() + + attr := "X-Attribute-User-Attribute" + content := "content of file" + valOne, valTwo := "first_value", "second_value" + fileName := "newFile.txt" + + var buff bytes.Buffer + w := multipart.NewWriter(&buff) + fw, err := w.CreateFormFile("file", fileName) + require.NoError(t, err) + _, err = io.Copy(fw, bytes.NewBufferString(content)) + require.NoError(t, err) + err = w.Close() + require.NoError(t, err) + + request, err := http.NewRequest(http.MethodPost, url, &buff) + require.NoError(t, err) + request.Header.Set("Content-Type", w.FormDataContentType()) + request.Header.Add(attr, valOne) + request.Header.Add(attr, valTwo) + + resp, err := http.DefaultClient.Do(request) + require.NoError(t, err) + + defer func() { + err := resp.Body.Close() + require.NoError(t, err) + }() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "key duplication error: "+attr+"\n", string(body)) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) +} + func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { content := "content of file" attributes := map[string]string{ diff --git a/uploader/filter.go b/uploader/filter.go index e453cfe..c7349c7 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -26,7 +26,8 @@ func systemTranslator(key, prefix []byte) []byte { return bytes.ToUpper(key) } -func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]string { +func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]string, error) { + var err error result := make(map[string]string) prefix := []byte(utils.UserAttributeHeaderPrefix) @@ -42,23 +43,30 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]str } // removing attribute prefix - key = bytes.TrimPrefix(key, prefix) + clearKey := bytes.TrimPrefix(key, prefix) // checks that it's a system NeoFS header for _, system := range neofsAttributeHeaderPrefixes { - if bytes.HasPrefix(key, system) { - key = systemTranslator(key, system) + if bytes.HasPrefix(clearKey, system) { + clearKey = systemTranslator(clearKey, system) break } } // checks that the attribute key is not empty - if len(key) == 0 { + if len(clearKey) == 0 { + return + } + + // check if key gets duplicated + // return error containing full key name (with prefix) + if _, ok := result[string(clearKey)]; ok { + err = fmt.Errorf("key duplication error: %s", string(key)) return } // make string representation of key / val - k, v := string(key), string(val) + k, v := string(clearKey), string(val) result[k] = v @@ -67,7 +75,7 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]str zap.String("val", v)) }) - return result + return result, err } func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations, now time.Time) error { diff --git a/uploader/filter_test.go b/uploader/filter_test.go index 585fac8..de4c541 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -16,8 +16,27 @@ import ( func TestFilter(t *testing.T) { log := zap.NewNop() + t.Run("duplicate keys error", func(t *testing.T) { + req := &fasthttp.RequestHeader{} + req.DisableNormalizing() + req.Add("X-Attribute-DupKey", "first-value") + req.Add("X-Attribute-DupKey", "second-value") + _, err := filterHeaders(log, req) + require.Error(t, err) + }) + + t.Run("duplicate system keys error", func(t *testing.T) { + req := &fasthttp.RequestHeader{} + req.DisableNormalizing() + req.Add("X-Attribute-Neofs-DupKey", "first-value") + req.Add("X-Attribute-Neofs-DupKey", "second-value") + _, err := filterHeaders(log, req) + require.Error(t, err) + }) + req := &fasthttp.RequestHeader{} req.DisableNormalizing() + req.Set("X-Attribute-Neofs-Expiration-Epoch1", "101") req.Set("X-Attribute-NEOFS-Expiration-Epoch2", "102") req.Set("X-Attribute-neofs-Expiration-Epoch3", "103") @@ -30,7 +49,8 @@ func TestFilter(t *testing.T) { "__NEOFS__EXPIRATION_EPOCH2": "102", } - result := filterHeaders(log, req) + result, err := filterHeaders(log, req) + require.NoError(t, err) require.Equal(t, expected, result) } diff --git a/uploader/upload.go b/uploader/upload.go index 8e65456..2effd16 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -114,7 +114,12 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } - filtered := filterHeaders(u.log, &c.Request.Header) + filtered, err := filterHeaders(u.log, &c.Request.Header) + if err != nil { + log.Error("could not process headers", zap.Error(err)) + response.Error(c, err.Error(), fasthttp.StatusBadRequest) + return + } if needParseExpiration(filtered) { epochDuration, err := getEpochDurations(c, u.pool) if err != nil { From f4fbd936bce0559303a75016d262440eac1794ef Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 24 Nov 2022 12:11:41 +0300 Subject: [PATCH 341/548] [#239] Remove deprecated linters Signed-off-by: Denis Kirillov --- .golangci.yml | 3 --- CHANGELOG.md | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index ebbff9d..67f93e7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -32,15 +32,12 @@ linters: - revive # some default golangci-lint linters - - deadcode - errcheck - gosimple - ineffassign - staticcheck - - structcheck - typecheck - unused - - varcheck # extra linters - exhaustive diff --git a/CHANGELOG.md b/CHANGELOG.md index 596457d..2ebef4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,19 @@ This document outlines major changes between releases. ## [Unreleased] +### Fixed +- Download zip archive when `FilePath` is invalid (#222) +- Only one peer must be healthy to init pool (#233) + ### Added +- Support the `Date` header on upload (#214) +- Add error response on attribute duplicates (#221) +- Timeout for individual operations in streaming RPC (#234) - Multiple server listeners (#228) +### Removed +- Deprecated linters (#239) + ### Updating from v0.25.0 Make sure your configuration is valid: @@ -20,15 +30,6 @@ If you configure application using `.yaml` file change: * `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`) * `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`) -### Fixed -- Download zip archive when `FilePath` is invalid (#222) -- Only one peer must be healthy to init pool (#233) - -### Added -- Support the `Date` header on upload (#214) -- Add error response on attribute duplicates (#221) -- Timeout for individual operations in streaming RPC (#234) - ## [0.25.0] - 2022-10-31 ### Added From f88fe1092d09fe1a0b79d07a41b5256a77011077 Mon Sep 17 00:00:00 2001 From: Dmitriy Zabolotskiy Date: Tue, 1 Nov 2022 11:29:14 +0300 Subject: [PATCH 342/548] [#223] Debian packaging Debian package includes: - user creation; - directories and permissions; - unit file for systemd Signed-off-by: Dmitriy Zabolotskiy --- .gitignore | 3 +++ Makefile | 19 +++++++++++++ debian/changelog | 5 ++++ debian/control | 15 +++++++++++ debian/copyright | 23 ++++++++++++++++ debian/neofs-http-gw.dirs | 2 ++ debian/neofs-http-gw.docs | 4 +++ debian/neofs-http-gw.examples | 1 + debian/neofs-http-gw.install | 2 ++ debian/neofs-http-gw.postinst | 51 +++++++++++++++++++++++++++++++++++ debian/neofs-http-gw.postrm | 41 ++++++++++++++++++++++++++++ debian/neofs-http-gw.preinst | 35 ++++++++++++++++++++++++ debian/neofs-http-gw.prerm | 38 ++++++++++++++++++++++++++ debian/neofs-http-gw.service | 16 +++++++++++ debian/rules | 16 +++++++++++ debian/source/format | 1 + docs/building-deb-package.md | 46 +++++++++++++++++++++++++++++++ 17 files changed, 318 insertions(+) create mode 100644 debian/changelog create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/neofs-http-gw.dirs create mode 100644 debian/neofs-http-gw.docs create mode 100644 debian/neofs-http-gw.examples create mode 100644 debian/neofs-http-gw.install create mode 100644 debian/neofs-http-gw.postinst create mode 100644 debian/neofs-http-gw.postrm create mode 100644 debian/neofs-http-gw.preinst create mode 100644 debian/neofs-http-gw.prerm create mode 100644 debian/neofs-http-gw.service create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100644 docs/building-deb-package.md diff --git a/.gitignore b/.gitignore index 2726d2d..ebadc7a 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ testfile coverage.txt coverage.html + +# debhelpers +**/.debhelper diff --git a/Makefile b/Makefile index 005fa28..3bea558 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,13 @@ BINS = $(BINDIR)/neofs-http-gw .PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint version clean +# .deb package versioning +OS_RELEASE = $(shell lsb_release -cs) +PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ + sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \ + sed "s/-/~/")-${OS_RELEASE} +.PHONY: debpackage debclean + # Make all binaries all: $(BINS) @@ -111,4 +118,16 @@ clean: rm -rf vendor rm -rf $(BINDIR) +# Package for Debian +debpackage: + dch --package neofs-http-gw \ + --controlmaint \ + --newversion $(PKG_VERSION) \ + --distribution $(OS_RELEASE) \ + "Please see CHANGELOG.md for code changes for $(VERSION)" + dpkg-buildpackage --no-sign -b + +debclean: + dh clean + include help.mk diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..eae76ab --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +neofs-http-gw (0.0.0) stable; urgency=medium + + * Please see CHANGELOG.md + + -- NeoSPCC Wed, 24 Aug 2022 18:29:49 +0300 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..e428567 --- /dev/null +++ b/debian/control @@ -0,0 +1,15 @@ +Source: neofs-http-gw +Section: neofs +Priority: optional +Maintainer: NeoSPCC +Build-Depends: debhelper-compat (= 13), dh-sysuser, git, devscripts +Standards-Version: 4.5.1 +Homepage: https://fs.neo.org/ +Vcs-Git: https://github.com/nspcc-dev/neofs-http-gw.git +Vcs-Browser: https://github.com/nspcc-dev/neofs-http-gw + +Package: neofs-http-gw +Architecture: any +Depends: ${misc:Depends} +Description: NeoFS HTTP Gateway bridges NeoFS internal protocol and HTTP standard. + diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..0849725 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,23 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: neofs-http-gw +Upstream-Contact: tech@nspcc.ru +Source: https://github.com/nspcc-dev/neofs-http-gw + +Files: * +Copyright: 2018-2022 NeoSPCC (@nspcc-dev), contributors of neofs-http-gw project + (https://github.com/nspcc-dev/neofs-http-gw/blob/master/CREDITS.md) + + +License: GPL-3 + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published + by the Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program or at /usr/share/common-licenses/GPL-3. + If not, see . diff --git a/debian/neofs-http-gw.dirs b/debian/neofs-http-gw.dirs new file mode 100644 index 0000000..9bacd8b --- /dev/null +++ b/debian/neofs-http-gw.dirs @@ -0,0 +1,2 @@ +etc/neofs +srv/neofs_cache diff --git a/debian/neofs-http-gw.docs b/debian/neofs-http-gw.docs new file mode 100644 index 0000000..884d34d --- /dev/null +++ b/debian/neofs-http-gw.docs @@ -0,0 +1,4 @@ +docs/gate-configuration.md +README.md +CREDITS.md +CONTRIBUTING.md diff --git a/debian/neofs-http-gw.examples b/debian/neofs-http-gw.examples new file mode 100644 index 0000000..dd04e98 --- /dev/null +++ b/debian/neofs-http-gw.examples @@ -0,0 +1 @@ +config/* diff --git a/debian/neofs-http-gw.install b/debian/neofs-http-gw.install new file mode 100644 index 0000000..afd9c36 --- /dev/null +++ b/debian/neofs-http-gw.install @@ -0,0 +1,2 @@ +bin/neofs-http-gw usr/bin +config/config.yaml etc/neofs/http diff --git a/debian/neofs-http-gw.postinst b/debian/neofs-http-gw.postinst new file mode 100644 index 0000000..ef4dbc0 --- /dev/null +++ b/debian/neofs-http-gw.postinst @@ -0,0 +1,51 @@ +#!/bin/sh +# postinst script for neofs-http-gw +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `configure' +# * `abort-upgrade' +# * `abort-remove' `in-favour' +# +# * `abort-remove' +# * `abort-deconfigure' `in-favour' +# `removing' +# +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + configure) + USERNAME=http + id -u neofs-$USERNAME >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /srv/neofs_cache --system -M -U -c "NeoFS HTTP gateway" neofs-$USERNAME + if ! dpkg-statoverride --list /etc/neofs/$USERNAME >/dev/null; then + chown -f root:neofs-$USERNAME /etc/neofs/$USERNAME + chown -f root:neofs-$USERNAME /etc/neofs/$USERNAME/config.yaml || true + chmod -f 0750 /etc/neofs/$USERNAME + chmod -f 0640 /etc/neofs/$USERNAME/config.yaml || true + fi + USERDIR=$(getent passwd "neofs-$USERNAME" | cut -d: -f6) + if ! dpkg-statoverride --list neofs-$USERDIR >/dev/null; then + chown -f neofs-$USERNAME: $USERDIR + fi + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/neofs-http-gw.postrm b/debian/neofs-http-gw.postrm new file mode 100644 index 0000000..02d5ccf --- /dev/null +++ b/debian/neofs-http-gw.postrm @@ -0,0 +1,41 @@ +#!/bin/sh +# postrm script for neofs-http-gw +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `remove' +# * `purge' +# * `upgrade' +# * `failed-upgrade' +# * `abort-install' +# * `abort-install' +# * `abort-upgrade' +# * `disappear' +# +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + purge) + rm -rf /srv/neofs_cache + ;; + + remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/neofs-http-gw.preinst b/debian/neofs-http-gw.preinst new file mode 100644 index 0000000..e832e16 --- /dev/null +++ b/debian/neofs-http-gw.preinst @@ -0,0 +1,35 @@ +#!/bin/sh +# preinst script for neofs-http-gw +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `install' +# * `install' +# * `upgrade' +# * `abort-upgrade' +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + install|upgrade) + ;; + + abort-upgrade) + ;; + + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/neofs-http-gw.prerm b/debian/neofs-http-gw.prerm new file mode 100644 index 0000000..10b59ec --- /dev/null +++ b/debian/neofs-http-gw.prerm @@ -0,0 +1,38 @@ +#!/bin/sh +# prerm script for neofs-http-gw +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `remove' +# * `upgrade' +# * `failed-upgrade' +# * `remove' `in-favour' +# * `deconfigure' `in-favour' +# `removing' +# +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + remove|upgrade|deconfigure) + ;; + + failed-upgrade) + ;; + + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/neofs-http-gw.service b/debian/neofs-http-gw.service new file mode 100644 index 0000000..d7ce5eb --- /dev/null +++ b/debian/neofs-http-gw.service @@ -0,0 +1,16 @@ +[Unit] +Description=NeoFS HTTP Gateway +Requires=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/neofs-http-gw --config /etc/neofs/http/config.yaml +User=neofs-http +Group=neofs-http +WorkingDirectory=/srv/neofs_cache +Restart=always +RestartSec=5 +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..cbeb2d1 --- /dev/null +++ b/debian/rules @@ -0,0 +1,16 @@ +#!/usr/bin/make -f + +# Do not try to strip Go binaries and do not run test +export DEB_BUILD_OPTIONS := nostrip nocheck +SERVICE = neofs-http-gw + +%: + dh $@ + +override_dh_installsystemd: + dh_installsystemd --no-enable --no-start $(SERVICE).service + +override_dh_installchangelogs: + dh_installchangelogs -k CHANGELOG.md + + diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/docs/building-deb-package.md b/docs/building-deb-package.md new file mode 100644 index 0000000..26a77a2 --- /dev/null +++ b/docs/building-deb-package.md @@ -0,0 +1,46 @@ +# Building Debian package on host + +## Prerequisites + +For now, we're assuming building for Debian 11 (stable) x86_64. + +Go version 18.4 or later should already be installed, i.e. this runs +successfully: + +* `make all` + +## Installing packaging dependencies + +```shell +$ sudo apt install debhelper-compat dh-sequence-bash-completion devscripts +``` + +Warining: number of package installed is pretty large considering dependecies. + +## Package building + +```shell +$ make debpackage +``` + +## Leftovers cleaning + +```shell +$ make debclean +``` +or +```shell +$ dh clean +``` + +# Package versioning + +By default, package version is based on product version and may also contain git +tags and hashes. + +Package version could be overwritten by setting `PKG_VERSION` variable before +build, Debian package versioning rules should be respected. + +```shell +$ PKG_VERSION=0.32.0 make debpackge +``` From 2d9eee81c252bdff7f7da875711410a5398c8bf1 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 14 Dec 2022 18:59:24 +0300 Subject: [PATCH 343/548] [#1] Build HTTP Gateway with FrostFS dependencies Signed-off-by: Alex Vanin --- app.go | 16 ++++++++-------- downloader/download.go | 22 +++++++++++----------- downloader/head.go | 12 ++++++------ go.mod | 15 ++++++++------- go.sum | 27 ++++++++++++++------------- integration_test.go | 16 ++++++++-------- metrics/metrics.go | 2 +- resolver/neofs.go | 2 +- resolver/resolver.go | 6 +++--- settings.go | 2 +- tokens/bearer-token.go | 2 +- tokens/bearer-token_test.go | 4 ++-- uploader/filter.go | 4 ++-- uploader/filter_test.go | 4 ++-- uploader/multipart.go | 2 +- uploader/upload.go | 18 +++++++++--------- utils/params.go | 6 +++--- utils/util.go | 4 ++-- 18 files changed, 83 insertions(+), 81 deletions(-) diff --git a/app.go b/app.go index 6eb0945..0cea989 100644 --- a/app.go +++ b/app.go @@ -11,20 +11,20 @@ import ( "sync" "syscall" + "github.com/TrueCloudLab/frostfs-http-gw/downloader" + "github.com/TrueCloudLab/frostfs-http-gw/metrics" + "github.com/TrueCloudLab/frostfs-http-gw/resolver" + "github.com/TrueCloudLab/frostfs-http-gw/response" + "github.com/TrueCloudLab/frostfs-http-gw/uploader" + "github.com/TrueCloudLab/frostfs-http-gw/utils" + "github.com/TrueCloudLab/frostfs-sdk-go/pool" + "github.com/TrueCloudLab/frostfs-sdk-go/user" "github.com/fasthttp/router" "github.com/nspcc-dev/neo-go/cli/flags" "github.com/nspcc-dev/neo-go/cli/input" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" - "github.com/nspcc-dev/neofs-http-gw/downloader" - "github.com/nspcc-dev/neofs-http-gw/metrics" - "github.com/nspcc-dev/neofs-http-gw/resolver" - "github.com/nspcc-dev/neofs-http-gw/response" - "github.com/nspcc-dev/neofs-http-gw/uploader" - "github.com/nspcc-dev/neofs-http-gw/utils" - "github.com/nspcc-dev/neofs-sdk-go/pool" - "github.com/nspcc-dev/neofs-sdk-go/user" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" diff --git a/downloader/download.go b/downloader/download.go index 296f58d..f34ab62 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -17,17 +17,17 @@ import ( "unicode" "unicode/utf8" - "github.com/nspcc-dev/neofs-http-gw/resolver" - "github.com/nspcc-dev/neofs-http-gw/response" - "github.com/nspcc-dev/neofs-http-gw/tokens" - "github.com/nspcc-dev/neofs-http-gw/utils" - "github.com/nspcc-dev/neofs-sdk-go/bearer" - "github.com/nspcc-dev/neofs-sdk-go/client" - "github.com/nspcc-dev/neofs-sdk-go/container" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" - "github.com/nspcc-dev/neofs-sdk-go/object" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/nspcc-dev/neofs-sdk-go/pool" + "github.com/TrueCloudLab/frostfs-http-gw/resolver" + "github.com/TrueCloudLab/frostfs-http-gw/response" + "github.com/TrueCloudLab/frostfs-http-gw/tokens" + "github.com/TrueCloudLab/frostfs-http-gw/utils" + "github.com/TrueCloudLab/frostfs-sdk-go/bearer" + "github.com/TrueCloudLab/frostfs-sdk-go/client" + "github.com/TrueCloudLab/frostfs-sdk-go/container" + cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/TrueCloudLab/frostfs-sdk-go/object" + oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/atomic" "go.uber.org/zap" diff --git a/downloader/head.go b/downloader/head.go index 3267228..2487b23 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -7,12 +7,12 @@ import ( "strings" "time" - "github.com/nspcc-dev/neofs-http-gw/response" - "github.com/nspcc-dev/neofs-http-gw/tokens" - "github.com/nspcc-dev/neofs-http-gw/utils" - "github.com/nspcc-dev/neofs-sdk-go/object" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/nspcc-dev/neofs-sdk-go/pool" + "github.com/TrueCloudLab/frostfs-http-gw/response" + "github.com/TrueCloudLab/frostfs-http-gw/tokens" + "github.com/TrueCloudLab/frostfs-http-gw/utils" + "github.com/TrueCloudLab/frostfs-sdk-go/object" + oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" ) diff --git a/go.mod b/go.mod index 87d1dc4..7d9c3be 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,12 @@ -module github.com/nspcc-dev/neofs-http-gw +module github.com/TrueCloudLab/frostfs-http-gw go 1.17 require ( + github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68 + github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221214065929-4c779423f556 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.99.4 - github.com/nspcc-dev/neofs-api-go/v2 v2.14.0 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221115140820-b4b07a3c4e11 github.com/prometheus/client_golang v1.13.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 @@ -21,6 +21,11 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/hcsshim v0.9.2 // indirect + github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42 // indirect + github.com/TrueCloudLab/frostfs-crypto v0.5.0 // indirect + github.com/TrueCloudLab/hrw v1.1.0 // indirect + github.com/TrueCloudLab/rfc6979 v0.3.0 // indirect + github.com/TrueCloudLab/tzhash v1.7.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -55,12 +60,8 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/hrw v1.0.9 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262 // indirect - github.com/nspcc-dev/neofs-contract v0.16.0 // indirect - github.com/nspcc-dev/neofs-crypto v0.4.0 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect - github.com/nspcc-dev/tzhash v1.6.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.1.1 // indirect diff --git a/go.sum b/go.sum index a4e3530..3e8348f 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,20 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68 h1:mwZr15qCuIcWojIOmH6LILPohbWIkknZe9vhBRapmfQ= +github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68/go.mod h1:u3P6aL/NpAIY5IFRsJhmV+61Q3pJ3BkLENqySkf5zZQ= +github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42 h1:P/gisZxTzJ9R3tuYDaQWe0tY6m1Zea3gzdPpNYK+NP4= +github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42/go.mod h1:qmf648elr+FWBZH3hqND8KVrXMnqu/e0z48k+sX8C2s= +github.com/TrueCloudLab/frostfs-crypto v0.5.0 h1:ZoLjixSkQv3j1EwZ1WJzMEJY2NR+9nO4Pd8WSyM/RRI= +github.com/TrueCloudLab/frostfs-crypto v0.5.0/go.mod h1:775MUewpH8AWpXrimAG2NYWOXB6lpKOI5kqgu+eI5zs= +github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221214065929-4c779423f556 h1:Cc1jjYxKPfyw7TIJh3Bje7m8DOSn2dx+2zmr0yusWGw= +github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221214065929-4c779423f556/go.mod h1:4ZiG4jNLzrqeJbmZUrPI7wDZhQVPaf0zEIWa/eBsqBg= +github.com/TrueCloudLab/hrw v1.1.0 h1:2U69PpUX1UtMWgh/RAg6D8mQW+/WsxbLNE+19EUhLhY= +github.com/TrueCloudLab/hrw v1.1.0/go.mod h1:Pzi8Hy3qx12cew+ajVxgbtDVM4sRG9/gJnJLcL/yRyY= +github.com/TrueCloudLab/rfc6979 v0.3.0 h1:0SYMAfQWh/TjnofqYQHy+s3rmQ5gi0fvOaDbqd60/Ic= +github.com/TrueCloudLab/rfc6979 v0.3.0/go.mod h1:qylxFXFQ/sMvpZC/8JyWp+mfzk5Zj/KDT5FAbekhobc= +github.com/TrueCloudLab/tzhash v1.7.0 h1:btGORepc7Dg+n4MxgJxv73c9eYhwSBI5HqsqUBRmJiw= +github.com/TrueCloudLab/tzhash v1.7.0/go.mod h1:gDQxqjhTqhR58Qfx0gxGtuyGAkixOukwbFGX9O6UGg4= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/ishell/v2 v2.0.2/go.mod h1:E4oTCXfo6QjoCart0QYa5m9w4S+deXs/P/9jA77A9Bs= @@ -738,41 +752,28 @@ github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= -github.com/nspcc-dev/dbft v0.0.0-20220629112714-fd49ca59d354/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/dbft v0.0.0-20220902113116-58a5e763e647/go.mod h1:g9xisXmX9NP9MjioaTe862n9SlZTrP+6PVUWLBYOr98= github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 h1:n4ZaFCKt1pQJd7PXoMJabZWK9ejjbLOVrkl/lOUmshg= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= -github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= -github.com/nspcc-dev/neo-go v0.99.2/go.mod h1:9P0yWqhZX7i/ChJ+zjtiStO1uPTolPFUM+L5oNznU8E= github.com/nspcc-dev/neo-go v0.99.4 h1:8Y+SdRxksC72a4PNkcGCh/aaQinh9Gu+c5LilbcsXOI= github.com/nspcc-dev/neo-go v0.99.4/go.mod h1:mKTolfRUfKjFso5HPvGSQtUZc70n0VKBMs16eGuC5gA= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220809123759-3094d3e0c14b/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262 h1:UTmSLZw5OpD/JPE1B5Vf98GF0zu2/Hsqq1lGLtStTUE= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.14.0 h1:jhuN8Ldqz7WApvUJRFY0bjRXE1R3iCkboMX5QVZhHVk= -github.com/nspcc-dev/neofs-api-go/v2 v2.14.0/go.mod h1:DRIr0Ic1s+6QgdqmNFNLIqMqd7lNMJfYwkczlm1hDtM= -github.com/nspcc-dev/neofs-contract v0.16.0 h1:/K5IMwZOlUCooxpe//73RW20HTfHvwRL088gppU5DM8= -github.com/nspcc-dev/neofs-contract v0.16.0/go.mod h1:gN5bo2TlMvLbySImmg76DVj3jVmYgti2VVlQ+h/tcr0= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-crypto v0.4.0 h1:5LlrUAM5O0k1+sH/sktBtrgfWtq1pgpDs09fZo+KYi4= github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221115140820-b4b07a3c4e11 h1:dcofGaVR6najaIdh0JOXoxqV1MLJWnJ3pPMhD5gowV4= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221115140820-b4b07a3c4e11/go.mod h1:7bH8zabbewpXstaXKoV0Tk3lV7KFYadi5edLU5yDERY= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= -github.com/nspcc-dev/tzhash v1.6.1 h1:8dUrWFpjkmoHF+7GxuGUmarj9LLHWFcuyF3CTrqq9JE= -github.com/nspcc-dev/tzhash v1.6.1/go.mod h1:BoflzCVp+DO/f1mvbcsJQWoFzidIFBhWFZMglbUW648= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= diff --git a/integration_test.go b/integration_test.go index d7cb8d0..3d6f14b 100644 --- a/integration_test.go +++ b/integration_test.go @@ -13,15 +13,15 @@ import ( "testing" "time" + "github.com/TrueCloudLab/frostfs-sdk-go/container" + "github.com/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/TrueCloudLab/frostfs-sdk-go/object" + oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/TrueCloudLab/frostfs-sdk-go/pool" + "github.com/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neofs-sdk-go/container" - "github.com/nspcc-dev/neofs-sdk-go/container/acl" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" - "github.com/nspcc-dev/neofs-sdk-go/netmap" - "github.com/nspcc-dev/neofs-sdk-go/object" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/nspcc-dev/neofs-sdk-go/pool" - "github.com/nspcc-dev/neofs-sdk-go/user" "github.com/spf13/viper" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" diff --git a/metrics/metrics.go b/metrics/metrics.go index 4520557..84b7114 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -3,7 +3,7 @@ package metrics import ( "net/http" - "github.com/nspcc-dev/neofs-sdk-go/pool" + "github.com/TrueCloudLab/frostfs-sdk-go/pool" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/zap" diff --git a/resolver/neofs.go b/resolver/neofs.go index 6931baa..308a060 100644 --- a/resolver/neofs.go +++ b/resolver/neofs.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "github.com/nspcc-dev/neofs-sdk-go/pool" + "github.com/TrueCloudLab/frostfs-sdk-go/pool" ) // NeoFSResolver represents virtual connection to the NeoFS network. diff --git a/resolver/resolver.go b/resolver/resolver.go index 6fdf948..686a147 100644 --- a/resolver/resolver.go +++ b/resolver/resolver.go @@ -6,9 +6,9 @@ import ( "fmt" "sync" - "github.com/nspcc-dev/neofs-sdk-go/container" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" - "github.com/nspcc-dev/neofs-sdk-go/ns" + "github.com/TrueCloudLab/frostfs-sdk-go/container" + cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/TrueCloudLab/frostfs-sdk-go/ns" ) const ( diff --git a/settings.go b/settings.go index bb841d0..526f48d 100644 --- a/settings.go +++ b/settings.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/nspcc-dev/neofs-http-gw/resolver" + "github.com/TrueCloudLab/frostfs-http-gw/resolver" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/valyala/fasthttp" diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index cf7dc45..e1d46a9 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -7,7 +7,7 @@ import ( "errors" "fmt" - "github.com/nspcc-dev/neofs-sdk-go/bearer" + "github.com/TrueCloudLab/frostfs-sdk-go/bearer" "github.com/valyala/fasthttp" ) diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index 59aabad..07df0f9 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -4,9 +4,9 @@ import ( "encoding/base64" "testing" + "github.com/TrueCloudLab/frostfs-sdk-go/bearer" + "github.com/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neofs-sdk-go/bearer" - "github.com/nspcc-dev/neofs-sdk-go/user" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" ) diff --git a/uploader/filter.go b/uploader/filter.go index c7349c7..0ee5b66 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -7,8 +7,8 @@ import ( "strconv" "time" - "github.com/nspcc-dev/neofs-api-go/v2/object" - "github.com/nspcc-dev/neofs-http-gw/utils" + "github.com/TrueCloudLab/frostfs-api-go/v2/object" + "github.com/TrueCloudLab/frostfs-http-gw/utils" "github.com/valyala/fasthttp" "go.uber.org/zap" ) diff --git a/uploader/filter_test.go b/uploader/filter_test.go index de4c541..dc6f462 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "github.com/nspcc-dev/neofs-api-go/v2/object" - "github.com/nspcc-dev/neofs-http-gw/utils" + "github.com/TrueCloudLab/frostfs-api-go/v2/object" + "github.com/TrueCloudLab/frostfs-http-gw/utils" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" "go.uber.org/zap" diff --git a/uploader/multipart.go b/uploader/multipart.go index 5928e60..ae03cc0 100644 --- a/uploader/multipart.go +++ b/uploader/multipart.go @@ -3,7 +3,7 @@ package uploader import ( "io" - "github.com/nspcc-dev/neofs-http-gw/uploader/multipart" + "github.com/TrueCloudLab/frostfs-http-gw/uploader/multipart" "go.uber.org/zap" ) diff --git a/uploader/upload.go b/uploader/upload.go index 2effd16..c60beef 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -9,15 +9,15 @@ import ( "strconv" "time" - "github.com/nspcc-dev/neofs-http-gw/resolver" - "github.com/nspcc-dev/neofs-http-gw/response" - "github.com/nspcc-dev/neofs-http-gw/tokens" - "github.com/nspcc-dev/neofs-http-gw/utils" - "github.com/nspcc-dev/neofs-sdk-go/bearer" - "github.com/nspcc-dev/neofs-sdk-go/object" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/nspcc-dev/neofs-sdk-go/pool" - "github.com/nspcc-dev/neofs-sdk-go/user" + "github.com/TrueCloudLab/frostfs-http-gw/resolver" + "github.com/TrueCloudLab/frostfs-http-gw/response" + "github.com/TrueCloudLab/frostfs-http-gw/tokens" + "github.com/TrueCloudLab/frostfs-http-gw/utils" + "github.com/TrueCloudLab/frostfs-sdk-go/bearer" + "github.com/TrueCloudLab/frostfs-sdk-go/object" + oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/TrueCloudLab/frostfs-sdk-go/pool" + "github.com/TrueCloudLab/frostfs-sdk-go/user" "github.com/valyala/fasthttp" "go.uber.org/atomic" "go.uber.org/zap" diff --git a/utils/params.go b/utils/params.go index 7c6c6eb..68d2921 100644 --- a/utils/params.go +++ b/utils/params.go @@ -1,9 +1,9 @@ package utils import ( - "github.com/nspcc-dev/neofs-http-gw/resolver" - "github.com/nspcc-dev/neofs-sdk-go/pool" - "github.com/nspcc-dev/neofs-sdk-go/user" + "github.com/TrueCloudLab/frostfs-http-gw/resolver" + "github.com/TrueCloudLab/frostfs-sdk-go/pool" + "github.com/TrueCloudLab/frostfs-sdk-go/user" "go.uber.org/zap" ) diff --git a/utils/util.go b/utils/util.go index 8255528..ac6edb4 100644 --- a/utils/util.go +++ b/utils/util.go @@ -3,8 +3,8 @@ package utils import ( "context" - "github.com/nspcc-dev/neofs-http-gw/resolver" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/TrueCloudLab/frostfs-http-gw/resolver" + cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" ) // GetContainerID decode container id, if it's not a valid container id From a014fb96fcf339e08c022ef92e070b6db08b1d10 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 16 Dec 2022 11:37:22 +0300 Subject: [PATCH 344/548] [#2] Update docs Signed-off-by: Denis Kirillov --- .github/CODEOWNERS | 2 +- CONTRIBUTING.md | 18 ++++---- README.md | 94 +++++++++++++++++++------------------- docs/api.md | 2 +- docs/gate-configuration.md | 10 ++-- 5 files changed, 63 insertions(+), 63 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1414545..37417c9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @alexvanin @masterSplinter01 @KirillovDenis +* @alexvanin @KirillovDenis diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5af026c..b365b64 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,8 +3,8 @@ First, thank you for contributing! We love and encourage pull requests from everyone. Please follow the guidelines: -- Check the open [issues](https://github.com/nspcc-dev/neofs-http-gw/issues) and - [pull requests](https://github.com/nspcc-dev/neofs-http-gw/pulls) for existing +- Check the open [issues](https://github.com/TrueCloudLab/frostfs-http-gw/issues) and + [pull requests](https://github.com/TrueCloudLab/frostfs-http-gw/pulls) for existing discussions. - Open an issue first, to discuss a new feature or enhancement. @@ -23,24 +23,24 @@ everyone. Please follow the guidelines: ## Development Workflow -Start by forking the `neofs-http-gw` repository, make changes in a branch and then +Start by forking the `frostfs-http-gw` repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: ### Set up your GitHub Repository -Fork [NeoFS HTTP Gateway -upstream](https://github.com/nspcc-dev/neofs-http-gw/fork) source repository +Fork [FrostFS HTTP Gateway +upstream](https://github.com/TrueCloudLab/frostfs-http-gw/fork) source repository to your own personal repository. Copy the URL of your fork (you will need it for the `git clone` command below). ```sh -$ git clone https://github.com/nspcc-dev/neofs-http-gw +$ git clone https://github.com/TrueCloudLab/frostfs-http-gw ``` ### Set up git remote as ``upstream`` ```sh -$ cd neofs-http-gw -$ git remote add upstream https://github.com/nspcc-dev/neofs-http-gw +$ cd frostfs-http-gw +$ git remote add upstream https://github.com/TrueCloudLab/frostfs-http-gw $ git fetch upstream $ git merge upstream/master ... @@ -107,7 +107,7 @@ contributors". To sign your work, just add a line like this at the end of your commit message: ``` -Signed-off-by: Samii Sakisaka +Signed-off-by: Samii Sakisaka ``` This can be easily done with the `--signoff` option to `git commit`. diff --git a/README.md b/README.md index 206d735..69ea624 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,30 @@

-NeoFS +FrostFS

- NeoFS is a decentralized distributed object storage integrated with the NEO Blockchain. + FrostFS is a decentralized distributed object storage integrated with the NEO Blockchain.

--- -[![Report](https://goreportcard.com/badge/github.com/nspcc-dev/neofs-http-gw)](https://goreportcard.com/report/github.com/nspcc-dev/neofs-http-gw) -![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/nspcc-dev/neofs-http-gw?sort=semver) -![License](https://img.shields.io/github/license/nspcc-dev/neofs-http-gw.svg?style=popout) +[![Report](https://goreportcard.com/badge/github.com/TrueCloudLab/frostfs-http-gw)](https://goreportcard.com/report/github.com/TrueCloudLab/frostfs-http-gw) +![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/TrueCloudLab/frostfs-http-gw?sort=semver) +![License](https://img.shields.io/github/license/TrueCloudLab/frostfs-http-gw.svg?style=popout) -# NeoFS HTTP Gateway +# FrostFS HTTP Gateway -NeoFS HTTP Gateway bridges NeoFS internal protocol and HTTP standard. -- you can download one file per request from the NeoFS Network -- you can upload one file per request into the NeoFS Network +FrostFS HTTP Gateway bridges FrostFS internal protocol and HTTP standard. +- you can download one file per request from the FrostFS Network +- you can upload one file per request into the FrostFS Network See available routes in [specification](./docs/api.md). ## Installation -```go install github.com/nspcc-dev/neofs-http-gw``` +```go install github.com/TrueCloudLab/frostfs-http-gw``` Or you can call `make` to build it from the cloned repository (the binary will -end up in `bin/neofs-http-gw`). To build neofs-http-gw binary in clean docker -environment, call `make docker/bin/neofs-http-gw`. +end up in `bin/frostfs-http-gw`). To build frostfs-http-gw binary in clean docker +environment, call `make docker/bin/frostfs-http-gw`. Other notable make targets: @@ -44,26 +44,26 @@ latest stable release). ## Execution -HTTP gateway itself is not a NeoFS node, so to access NeoFS it uses node's +HTTP gateway itself is not a FrostFS node, so to access FrostFS it uses node's gRPC interface and you need to provide some node that it will connect to. This can be done either via `-p` parameter or via `HTTP_GW_PEERS__ADDRESS` and `HTTP_GW_PEERS__WEIGHT` environment variables (the gate supports multiple -NeoFS nodes with weighted load balancing). +FrostFS nodes with weighted load balancing). -If you launch HTTP gateway in bundle with [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env), +If you launch HTTP gateway in bundle with [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env), you can get the IP address of the node in the output of `make hosts` command (with s0*.neofs.devenv name). These two commands are functionally equivalent, they run the gate with one backend node (and otherwise default settings): ``` -$ neofs-http-gw -p 192.168.130.72:8080 -$ HTTP_GW_PEERS_0_ADDRESS=192.168.130.72:8080 neofs-http-gw +$ frostfs-http-gw -p 192.168.130.72:8080 +$ HTTP_GW_PEERS_0_ADDRESS=192.168.130.72:8080 frostfs-http-gw ``` It's also possible to specify uri scheme (grpc or grpcs) when using `-p`: ``` -$ neofs-http-gw -p grpc://192.168.130.72:8080 -$ HTTP_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 neofs-http-gw +$ frostfs-http-gw -p grpc://192.168.130.72:8080 +$ HTTP_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 frostfs-http-gw ``` ## Configuration @@ -74,11 +74,11 @@ environment variables (see [example](./config/config.env)), so they're not speci ### Nodes: weights and priorities -You can specify multiple `-p` options to add more NeoFS nodes, this will make +You can specify multiple `-p` options to add more FrostFS nodes, this will make gateway spread requests equally among them (using weight 1 and priority 1 for every node): ``` -$ neofs-http-gw -p 192.168.130.72:8080 -p 192.168.130.71:8080 +$ frostfs-http-gw -p 192.168.130.72:8080 -p 192.168.130.71:8080 ``` If you want some specific load distribution proportions, use weights and priorities: @@ -86,7 +86,7 @@ If you want some specific load distribution proportions, use weights and priorit $ HTTP_GW_PEERS_0_ADDRESS=192.168.130.71:8080 HTTP_GW_PEERS_0_WEIGHT=1 HTTP_GW_PEERS_0_PRIORITY=1 \ HTTP_GW_PEERS_1_ADDRESS=192.168.130.72:8080 HTTP_GW_PEERS_1_WEIGHT=9 HTTP_GW_PEERS_1_PRIORITY=2 \ HTTP_GW_PEERS_2_ADDRESS=192.168.130.73:8080 HTTP_GW_PEERS_2_WEIGHT=1 HTTP_GW_PEERS_2_PRIORITY=2 \ - neofs-http-gw + frostfs-http-gw ``` This command will make gateway use 192.168.130.71 while it is healthy. Otherwise, it will make the gateway use 192.168.130.72 for 90% of requests and 192.168.130.73 for remaining 10%. @@ -94,13 +94,13 @@ This command will make gateway use 192.168.130.71 while it is healthy. Otherwise ### Keys You can provide a wallet via `--wallet` or `-w` flag. You can also specify the account address using `--address` (if no address provided default one will be used). If wallet is used, you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt the wallet. -If no wallet provided, the gateway autogenerates a key pair it will use for NeoFS requests. +If no wallet provided, the gateway autogenerates a key pair it will use for FrostFS requests. ``` -$ neofs-http-gw -p $NEOFS_NODE -w $WALLET_PATH --address $ACCOUNT_ADDRESS +$ frostfs-http-gw -p $NEOFS_NODE -w $WALLET_PATH --address $ACCOUNT_ADDRESS ``` Example: ``` -$ neofs-http-gw -p 192.168.130.72:8080 -w wallet.json --address NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP +$ frostfs-http-gw -p 192.168.130.72:8080 -w wallet.json --address NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP ``` ### Binding and TLS @@ -116,7 +116,7 @@ external redirecting solution. Example to bind to `192.168.130.130:443` and serve TLS there: ``` -$ neofs-http-gw -p 192.168.130.72:8080 --listen_address 192.168.130.130:443 \ +$ frostfs-http-gw -p 192.168.130.72:8080 --listen_address 192.168.130.130:443 \ --tls_key=key.pem --tls_certificate=cert.pem ``` @@ -134,12 +134,12 @@ request with data stream after timeout. `HTTP_GW_WEB_STREAM_REQUEST_BODY` environment variable can be used to disable request body streaming (effectively it'll make the gateway accept the file completely -first and only then try sending it to NeoFS). +first and only then try sending it to FrostFS). `HTTP_GW_WEB_MAX_REQUEST_BODY_SIZE` controls maximum request body size limiting uploads to files slightly lower than this limit. -### NeoFS parameters +### FrostFS parameters Gateway can automatically set timestamps for uploaded files based on local time source, use `HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP` environment @@ -177,7 +177,7 @@ HTTP_GW_LOGGER_LEVEL=debug Configuration file is optional and can be used instead of environment variables/other parameters. It can be specified with `--config` parameter: ``` -$ neofs-http-gw --config your-config.yaml +$ frostfs-http-gw --config your-config.yaml ``` See [config](./config/config.yaml) and [defaults](./docs/gate-configuration.md) for example. @@ -185,7 +185,7 @@ See [config](./config/config.yaml) and [defaults](./docs/gate-configuration.md) ## HTTP API provided This gateway intentionally provides limited feature set and doesn't try to -substitute (or completely wrap) regular gRPC NeoFS interface. You can download +substitute (or completely wrap) regular gRPC FrostFS interface. You can download and upload objects with it, but deleting, searching, managing ACLs, creating containers and other activities are not supported and not planned to be supported. @@ -211,7 +211,7 @@ resolve_order: - nns ``` -2. Make sure your container is registered in NNS contract. If you use [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env) +2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env) you can check if your container (e.g. with `container-name` name) is registered in NNS: ```shell @@ -238,9 +238,9 @@ $ curl http://localhost:8082/get_by_attribute/container-name/FileName/object-nam #### Create a container -You can create a container via [neofs-cli](https://github.com/nspcc-dev/neofs-node/releases): +You can create a container via [frostfs-cli](https://github.com/TrueCloudLab/frostfs-node/releases): ``` -$ neofs-cli -r $NEOFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL +$ frostfs-cli -r $NEOFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL ``` where `$WALLET` is a path to user wallet, `$ACL` -- hex encoded basic ACL value or keywords 'private, 'public-read', 'public-read-write' and @@ -248,18 +248,18 @@ where `$WALLET` is a path to user wallet, For example: ``` -$ neofs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy "REP 3" --basic-acl public --await +$ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy "REP 3" --basic-acl public --await ``` -If you have launched nodes via [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env), +If you have launched nodes via [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env), you can get the key value from `wallets/wallet.json` or write the path to the file `wallets/wallet.key`. #### Prepare a file in a container -To create a file via [neofs-cli](https://github.com/nspcc-dev/neofs-node/releases), run a command below: +To create a file via [frostfs-cli](https://github.com/TrueCloudLab/frostfs-node/releases), run a command below: ``` -$ neofs-cli -r $NEOFS_NODE -k $KEY object put --file $FILENAME --cid $CID +$ frostfs-cli -r $NEOFS_NODE -k $KEY object put --file $FILENAME --cid $CID ``` where `$KEY` -- the key, please read the information [above](#create-a-container), @@ -267,7 +267,7 @@ where For example: ``` -$ neofs-cli -r 192.168.130.72:8080 -w ./wallet.json object put --file cat.png --cid Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ --attributes img_type=cat,my_attr=cute +$ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json object put --file cat.png --cid Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ --attributes img_type=cat,my_attr=cute ``` @@ -372,7 +372,7 @@ set of reply headers generated using the following rules: * `x-container-id` contains container ID * `x-object-id` contains object ID * `x-owner-id` contains owner address - * all the other NeoFS attributes are converted to `X-Attribute-*` headers (but only + * all the other FrostFS attributes are converted to `X-Attribute-*` headers (but only if they can be safely represented in HTTP header), for example `FileName` attribute becomes `X-Attribute-FileName` header @@ -452,25 +452,25 @@ operations for a request signed with your HTTP Gateway keys. If your don't want to manage gateway's secret keys and adjust eACL rules when gateway configuration changes (new gate, key rotation, etc) or you plan to use public services, there is an option to let your application backend (or you) to -issue Bearer Tokens ans pass them from the client via gate down to NeoFS level +issue Bearer Tokens ans pass them from the client via gate down to FrostFS level to grant access. -NeoFS Bearer Token basically is a container owner-signed ACL data (refer to NeoFS +FrostFS Bearer Token basically is a container owner-signed ACL data (refer to FrostFS documentation for more details). There are two options to pass them to gateway: * "Authorization" header with "Bearer" type and base64-encoded token in credentials field * "Bearer" cookie with base64-encoded token contents For example, you have a mobile application frontend with a backend part storing -data in NeoFS. When a user authorizes in the mobile app, the backend issues a NeoFS +data in FrostFS. When a user authorizes in the mobile app, the backend issues a FrostFS Bearer token and provides it to the frontend. Then, the mobile app may generate -some data and upload it via any available NeoFS HTTP Gateway by adding +some data and upload it via any available FrostFS HTTP Gateway by adding the corresponding header to the upload request. Accessing the ACL protected data works the same way. ##### Example In order to generate a bearer token, you need to know the container owner key and -the address of the sender who will do the request to NeoFS (in our case, it's a gateway wallet address). +the address of the sender who will do the request to FrostFS (in our case, it's a gateway wallet address). Suppose we have: * **KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr** (container owner key) @@ -521,7 +521,7 @@ Now, we can form a Bearer token (10000 is liftetime expiration in epoch) and sav Next, sign it with the container owner key: ``` -$ neofs-cli util sign bearer-token --from bearer.json --to signed.json -w ./wallet.json +$ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w ./wallet.json ``` Encoding to base64 to use via the header: ``` @@ -548,12 +548,12 @@ For the token to work correctly, you need to create a container with a basic ACL For example: ``` -$ neofs-cli -w ./wallet.json --basic-acl 0x0FFFCFFF -r 192.168.130.72:8080 container create --policy "REP 3" --await +$ frostfs-cli -w ./wallet.json --basic-acl 0x0FFFCFFF -r 192.168.130.72:8080 container create --policy "REP 3" --await ``` To deny access to a container without a token, set the eACL rules: ``` -$ neofs-cli -w ./wallet.json -r 192.168.130.72:8080 container set-eacl --table eacl.json --await --cid BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K +$ frostfs-cli -w ./wallet.json -r 192.168.130.72:8080 container set-eacl --table eacl.json --await --cid BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K ``` File **eacl.json**: diff --git a/docs/api.md b/docs/api.md index c7dd48b..6a19465 100644 --- a/docs/api.md +++ b/docs/api.md @@ -50,7 +50,7 @@ Route: `/upload/{cid}` #### POST -Upload file as object with attributes to NeoFS. +Upload file as object with attributes to FrostFS. ##### Request diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 5944e39..2f65670 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -1,6 +1,6 @@ -# NeoFS HTTP Gateway configuration file +# FrostFS HTTP Gateway configuration file -This section contains detailed NeoFS HTTP Gateway configuration file description +This section contains detailed FrostFS HTTP Gateway configuration file description including default config values and some tips to set up configurable values. There are some custom types used for brevity: @@ -23,19 +23,19 @@ $ kill -s SIGHUP Example: ```shell -$ ./bin/neofs-http-gw --config config.yaml &> http.log & +$ ./bin/frostfs-http-gw --config config.yaml &> http.log & [1] 998346 $ cat http.log # ... -2022-10-03T09:37:25.826+0300 info neofs-http-gw/app.go:332 starting application {"app_name": "neofs-http-gw", "version": "v0.24.0"} +2022-10-03T09:37:25.826+0300 info frostfs-http-gw/app.go:332 starting application {"app_name": "frostfs-http-gw", "version": "v0.24.0"} # ... $ kill -s SIGHUP 998346 $ cat http.log # ... -2022-10-03T09:38:16.205+0300 info neofs-http-gw/app.go:470 SIGHUP config reload completed +2022-10-03T09:38:16.205+0300 info frostfs-http-gw/app.go:470 SIGHUP config reload completed ``` # Structure From 31d396a125963595b3b60f9d92a993d48b33c6c5 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 16 Dec 2022 11:45:27 +0300 Subject: [PATCH 345/548] [#2] Update building Signed-off-by: Denis Kirillov --- Dockerfile | 4 ++-- Dockerfile.dirty | 4 ++-- Makefile | 10 +++++----- README.md | 2 +- app.go | 4 ++-- settings.go | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index b3ac08f..189dc22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,6 @@ FROM scratch WORKDIR / COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=builder /src/bin/neofs-http-gw /bin/neofs-http-gw +COPY --from=builder /src/bin/frostfs-http-gw /bin/frostfs-http-gw -ENTRYPOINT ["/bin/neofs-http-gw"] +ENTRYPOINT ["/bin/frostfs-http-gw"] diff --git a/Dockerfile.dirty b/Dockerfile.dirty index c17734b..f733447 100644 --- a/Dockerfile.dirty +++ b/Dockerfile.dirty @@ -3,6 +3,6 @@ RUN apk add --update --no-cache bash ca-certificates WORKDIR / -COPY bin/neofs-http-gw /bin/neofs-http-gw +COPY bin/frostfs-http-gw /bin/frostfs-http-gw -CMD ["neofs-http-gw"] +CMD ["frostfs-http-gw"] diff --git a/Makefile b/Makefile index 3bea558..41abb24 100644 --- a/Makefile +++ b/Makefile @@ -6,13 +6,13 @@ GO_VERSION ?= 1.19 LINT_VERSION ?= 1.49.0 BUILD ?= $(shell date -u --iso=seconds) -HUB_IMAGE ?= nspccdev/neofs-http-gw +HUB_IMAGE ?= truecloudlab/frostfs-http-gw HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" # List of binaries to build. For now just one. BINDIR = bin DIRS = $(BINDIR) -BINS = $(BINDIR)/neofs-http-gw +BINS = $(BINDIR)/frostfs-http-gw .PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint version clean @@ -74,7 +74,7 @@ fmt: # Build clean Docker image image: - @echo "⇒ Build NeoFS HTTP Gateway docker image " + @echo "⇒ Build FrostFS HTTP Gateway docker image " @docker build \ --build-arg REPO=$(REPO) \ --build-arg VERSION=$(VERSION) \ @@ -89,7 +89,7 @@ image-push: # Build dirty Docker image dirty-image: - @echo "⇒ Build NeoFS HTTP Gateway dirty docker image " + @echo "⇒ Build FrostFS HTTP Gateway dirty docker image " @docker build \ --build-arg REPO=$(REPO) \ --build-arg VERSION=$(VERSION) \ @@ -120,7 +120,7 @@ clean: # Package for Debian debpackage: - dch --package neofs-http-gw \ + dch --package frostfs-http-gw \ --controlmaint \ --newversion $(PKG_VERSION) \ --distribution $(OS_RELEASE) \ diff --git a/README.md b/README.md index 69ea624..0c05b60 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ version Show current version ``` Or you can also use a [Docker -image](https://hub.docker.com/r/nspccdev/neofs-http-gw) provided for the released +image](https://hub.docker.com/r/truecloudlab/frostfs-http-gw) provided for the released (and occasionally unreleased) versions of the gateway (`:latest` points to the latest stable release). diff --git a/app.go b/app.go index 0cea989..80c36f0 100644 --- a/app.go +++ b/app.go @@ -111,7 +111,7 @@ func newApp(ctx context.Context, opt ...Option) App { } // -- setup FastHTTP server -- - a.webServer.Name = "neofs-http-gw" + a.webServer.Name = "frost-http-gw" a.webServer.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) a.webServer.WriteBufferSize = a.cfg.GetInt(cfgWebWriteBufferSize) a.webServer.ReadTimeout = a.cfg.GetDuration(cfgWebReadTimeout) @@ -327,7 +327,7 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*ecds } func (a *app) Wait() { - a.log.Info("starting application", zap.String("app_name", "neofs-http-gw"), zap.String("version", Version)) + a.log.Info("starting application", zap.String("app_name", "frostfs-http-gw"), zap.String("version", Version)) a.setHealthStatus() diff --git a/settings.go b/settings.go index 526f48d..c6ee33f 100644 --- a/settings.go +++ b/settings.go @@ -123,7 +123,7 @@ func settings() *viper.Viper { flags.String(cmdListenAddress, "0.0.0.0:8080", "addresses to listen") flags.String(cfgTLSCertFile, "", "TLS certificate path") flags.String(cfgTLSKeyFile, "", "TLS key path") - peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") + peers := flags.StringArrayP(cfgPeers, "p", nil, "FrostFS nodes") resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order") @@ -197,7 +197,7 @@ func settings() *viper.Viper { switch { case help != nil && *help: - fmt.Printf("NeoFS HTTP Gateway %s\n", Version) + fmt.Printf("FrostFS HTTP Gateway %s\n", Version) flags.PrintDefaults() fmt.Println() @@ -229,7 +229,7 @@ func settings() *viper.Viper { os.Exit(0) case version != nil && *version: - fmt.Printf("NeoFS HTTP Gateway\nVersion: %s\nGoVersion: %s\n", Version, runtime.Version()) + fmt.Printf("FrostFS HTTP Gateway\nVersion: %s\nGoVersion: %s\n", Version, runtime.Version()) os.Exit(0) } From de309e3264f733811720f322ef78ed98e7304557 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 16 Dec 2022 11:54:19 +0300 Subject: [PATCH 346/548] [#2] Update deb packages Signed-off-by: Denis Kirillov --- debian/changelog | 4 ++-- debian/control | 16 +++++++-------- debian/copyright | 10 ++++++---- debian/frostfs-http-gw.dirs | 2 ++ ...eofs-http-gw.docs => frostfs-http-gw.docs} | 0 ...p-gw.examples => frostfs-http-gw.examples} | 0 debian/frostfs-http-gw.install | 2 ++ ...p-gw.postinst => frostfs-http-gw.postinst} | 20 +++++++++---------- ...-http-gw.postrm => frostfs-http-gw.postrm} | 4 ++-- ...ttp-gw.preinst => frostfs-http-gw.preinst} | 2 +- ...fs-http-gw.prerm => frostfs-http-gw.prerm} | 2 +- debian/frostfs-http-gw.service | 16 +++++++++++++++ debian/neofs-http-gw.dirs | 2 -- debian/neofs-http-gw.install | 2 -- debian/neofs-http-gw.service | 16 --------------- debian/rules | 2 +- 16 files changed, 51 insertions(+), 49 deletions(-) create mode 100644 debian/frostfs-http-gw.dirs rename debian/{neofs-http-gw.docs => frostfs-http-gw.docs} (100%) rename debian/{neofs-http-gw.examples => frostfs-http-gw.examples} (100%) create mode 100644 debian/frostfs-http-gw.install rename debian/{neofs-http-gw.postinst => frostfs-http-gw.postinst} (56%) rename debian/{neofs-http-gw.postrm => frostfs-http-gw.postrm} (92%) rename debian/{neofs-http-gw.preinst => frostfs-http-gw.preinst} (94%) rename debian/{neofs-http-gw.prerm => frostfs-http-gw.prerm} (96%) create mode 100644 debian/frostfs-http-gw.service delete mode 100644 debian/neofs-http-gw.dirs delete mode 100644 debian/neofs-http-gw.install delete mode 100644 debian/neofs-http-gw.service diff --git a/debian/changelog b/debian/changelog index eae76ab..257bf0f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -neofs-http-gw (0.0.0) stable; urgency=medium +frostfs-http-gw (0.0.0) stable; urgency=medium * Please see CHANGELOG.md - -- NeoSPCC Wed, 24 Aug 2022 18:29:49 +0300 + -- TrueCloudLab Wed, 24 Aug 2022 18:29:49 +0300 diff --git a/debian/control b/debian/control index e428567..cd18ad5 100644 --- a/debian/control +++ b/debian/control @@ -1,15 +1,15 @@ -Source: neofs-http-gw -Section: neofs +Source: frostfs-http-gw +Section: frostfs Priority: optional -Maintainer: NeoSPCC +Maintainer: TrueCloudLab Build-Depends: debhelper-compat (= 13), dh-sysuser, git, devscripts Standards-Version: 4.5.1 -Homepage: https://fs.neo.org/ -Vcs-Git: https://github.com/nspcc-dev/neofs-http-gw.git -Vcs-Browser: https://github.com/nspcc-dev/neofs-http-gw +Homepage: https://frostfs.info/ +Vcs-Git: https://github.com/TrueCloudLab/frostfs-http-gw.git +Vcs-Browser: https://github.com/TrueCloudLab/frostfs-http-gw -Package: neofs-http-gw +Package: frostfs-http-gw Architecture: any Depends: ${misc:Depends} -Description: NeoFS HTTP Gateway bridges NeoFS internal protocol and HTTP standard. +Description: FrostFS HTTP Gateway bridges FrostFS internal protocol and HTTP standard. diff --git a/debian/copyright b/debian/copyright index 0849725..be82996 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,11 +1,13 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: neofs-http-gw -Upstream-Contact: tech@nspcc.ru -Source: https://github.com/nspcc-dev/neofs-http-gw +Upstream-Name: frostfs-http-gw +Upstream-Contact: tech@frostfs.info +Source: https://github.com/TrueCloudLab/frostfs-http-gw Files: * -Copyright: 2018-2022 NeoSPCC (@nspcc-dev), contributors of neofs-http-gw project +Copyright: 2018-2022 NeoSPCC (@nspcc-dev), contributors of neofs-http-gw project (https://github.com/nspcc-dev/neofs-http-gw/blob/master/CREDITS.md) + 2022 True Cloud Lab (@TrueCloudLab), contributors of frostfs-http-gw project + (https://github.com/TrueCloudLab/frostfs-http-gw/blob/master/CREDITS.md) License: GPL-3 diff --git a/debian/frostfs-http-gw.dirs b/debian/frostfs-http-gw.dirs new file mode 100644 index 0000000..f51b198 --- /dev/null +++ b/debian/frostfs-http-gw.dirs @@ -0,0 +1,2 @@ +etc/frostfs +srv/frostfs_cache diff --git a/debian/neofs-http-gw.docs b/debian/frostfs-http-gw.docs similarity index 100% rename from debian/neofs-http-gw.docs rename to debian/frostfs-http-gw.docs diff --git a/debian/neofs-http-gw.examples b/debian/frostfs-http-gw.examples similarity index 100% rename from debian/neofs-http-gw.examples rename to debian/frostfs-http-gw.examples diff --git a/debian/frostfs-http-gw.install b/debian/frostfs-http-gw.install new file mode 100644 index 0000000..2f71be4 --- /dev/null +++ b/debian/frostfs-http-gw.install @@ -0,0 +1,2 @@ +bin/frostfs-http-gw usr/bin +config/config.yaml etc/frostfs/http diff --git a/debian/neofs-http-gw.postinst b/debian/frostfs-http-gw.postinst similarity index 56% rename from debian/neofs-http-gw.postinst rename to debian/frostfs-http-gw.postinst index ef4dbc0..85b6c0f 100644 --- a/debian/neofs-http-gw.postinst +++ b/debian/frostfs-http-gw.postinst @@ -1,5 +1,5 @@ #!/bin/sh -# postinst script for neofs-http-gw +# postinst script for frostfs-http-gw # # see: dh_installdeb(1) @@ -21,16 +21,16 @@ set -e case "$1" in configure) USERNAME=http - id -u neofs-$USERNAME >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /srv/neofs_cache --system -M -U -c "NeoFS HTTP gateway" neofs-$USERNAME - if ! dpkg-statoverride --list /etc/neofs/$USERNAME >/dev/null; then - chown -f root:neofs-$USERNAME /etc/neofs/$USERNAME - chown -f root:neofs-$USERNAME /etc/neofs/$USERNAME/config.yaml || true - chmod -f 0750 /etc/neofs/$USERNAME - chmod -f 0640 /etc/neofs/$USERNAME/config.yaml || true + id -u frostfs-$USERNAME >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /srv/frostfs_cache --system -M -U -c "FrostFS HTTP gateway" frostfs-$USERNAME + if ! dpkg-statoverride --list /etc/frostfs/$USERNAME >/dev/null; then + chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME + chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/config.yaml || true + chmod -f 0750 /etc/frostfs/$USERNAME + chmod -f 0640 /etc/frostfs/$USERNAME/config.yaml || true fi - USERDIR=$(getent passwd "neofs-$USERNAME" | cut -d: -f6) - if ! dpkg-statoverride --list neofs-$USERDIR >/dev/null; then - chown -f neofs-$USERNAME: $USERDIR + USERDIR=$(getent passwd "frostfs-$USERNAME" | cut -d: -f6) + if ! dpkg-statoverride --list frostfs-$USERDIR >/dev/null; then + chown -f frostfs-$USERNAME: $USERDIR fi ;; diff --git a/debian/neofs-http-gw.postrm b/debian/frostfs-http-gw.postrm similarity index 92% rename from debian/neofs-http-gw.postrm rename to debian/frostfs-http-gw.postrm index 02d5ccf..ebb2dec 100644 --- a/debian/neofs-http-gw.postrm +++ b/debian/frostfs-http-gw.postrm @@ -1,5 +1,5 @@ #!/bin/sh -# postrm script for neofs-http-gw +# postrm script for frostfs-http-gw # # see: dh_installdeb(1) @@ -21,7 +21,7 @@ set -e case "$1" in purge) - rm -rf /srv/neofs_cache + rm -rf /srv/frostfs_cache ;; remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) diff --git a/debian/neofs-http-gw.preinst b/debian/frostfs-http-gw.preinst similarity index 94% rename from debian/neofs-http-gw.preinst rename to debian/frostfs-http-gw.preinst index e832e16..c18093b 100644 --- a/debian/neofs-http-gw.preinst +++ b/debian/frostfs-http-gw.preinst @@ -1,5 +1,5 @@ #!/bin/sh -# preinst script for neofs-http-gw +# preinst script for frostfs-http-gw # # see: dh_installdeb(1) diff --git a/debian/neofs-http-gw.prerm b/debian/frostfs-http-gw.prerm similarity index 96% rename from debian/neofs-http-gw.prerm rename to debian/frostfs-http-gw.prerm index 10b59ec..7623f7f 100644 --- a/debian/neofs-http-gw.prerm +++ b/debian/frostfs-http-gw.prerm @@ -1,5 +1,5 @@ #!/bin/sh -# prerm script for neofs-http-gw +# prerm script for frostfs-http-gw # # see: dh_installdeb(1) diff --git a/debian/frostfs-http-gw.service b/debian/frostfs-http-gw.service new file mode 100644 index 0000000..4851a3f --- /dev/null +++ b/debian/frostfs-http-gw.service @@ -0,0 +1,16 @@ +[Unit] +Description=FrostFS HTTP Gateway +Requires=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/frostfs-http-gw --config /etc/frostfs/http/config.yaml +User=frostfs-http +Group=frostfs-http +WorkingDirectory=/srv/frostfs_cache +Restart=always +RestartSec=5 +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/debian/neofs-http-gw.dirs b/debian/neofs-http-gw.dirs deleted file mode 100644 index 9bacd8b..0000000 --- a/debian/neofs-http-gw.dirs +++ /dev/null @@ -1,2 +0,0 @@ -etc/neofs -srv/neofs_cache diff --git a/debian/neofs-http-gw.install b/debian/neofs-http-gw.install deleted file mode 100644 index afd9c36..0000000 --- a/debian/neofs-http-gw.install +++ /dev/null @@ -1,2 +0,0 @@ -bin/neofs-http-gw usr/bin -config/config.yaml etc/neofs/http diff --git a/debian/neofs-http-gw.service b/debian/neofs-http-gw.service deleted file mode 100644 index d7ce5eb..0000000 --- a/debian/neofs-http-gw.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=NeoFS HTTP Gateway -Requires=network.target - -[Service] -Type=simple -ExecStart=/usr/bin/neofs-http-gw --config /etc/neofs/http/config.yaml -User=neofs-http -Group=neofs-http -WorkingDirectory=/srv/neofs_cache -Restart=always -RestartSec=5 -PrivateTmp=true - -[Install] -WantedBy=multi-user.target diff --git a/debian/rules b/debian/rules index cbeb2d1..477128a 100755 --- a/debian/rules +++ b/debian/rules @@ -2,7 +2,7 @@ # Do not try to strip Go binaries and do not run test export DEB_BUILD_OPTIONS := nostrip nocheck -SERVICE = neofs-http-gw +SERVICE = frostfs-http-gw %: dh $@ From 67c5818fc1b4c13613cac9b0e0399225fe4c5809 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 20 Dec 2022 14:01:50 +0300 Subject: [PATCH 347/548] [#2] Rename internals Signed-off-by: Denis Kirillov --- .gitignore | 2 +- README.md | 14 +++++++------- app.go | 8 ++++---- config/config.env | 12 ++++++------ config/config.yaml | 12 ++++++------ docs/gate-configuration.md | 14 +++++++------- downloader/download.go | 6 +++--- downloader/head.go | 4 ++-- metrics/metrics.go | 2 +- resolver/frostfs.go | 35 +++++++++++++++++++++++++++++++++++ resolver/neofs.go | 35 ----------------------------------- resolver/resolver.go | 18 +++++++++--------- uploader/filter.go | 4 ++-- uploader/upload.go | 4 ++-- 14 files changed, 85 insertions(+), 85 deletions(-) create mode 100644 resolver/frostfs.go delete mode 100644 resolver/neofs.go diff --git a/.gitignore b/.gitignore index ebadc7a..c4a98d8 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ temp test.sh testfile .blast.yml -.neofs-cli.yml +.frostfs-cli.yml .cache diff --git a/README.md b/README.md index 0c05b60..e336a6d 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ FrostFS nodes with weighted load balancing). If you launch HTTP gateway in bundle with [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env), you can get the IP address of the node in the output of `make hosts` command -(with s0*.neofs.devenv name). +(with s0*.frostfs.devenv name). These two commands are functionally equivalent, they run the gate with one backend node (and otherwise default settings): @@ -96,7 +96,7 @@ You can provide a wallet via `--wallet` or `-w` flag. You can also specify the a (if no address provided default one will be used). If wallet is used, you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt the wallet. If no wallet provided, the gateway autogenerates a key pair it will use for FrostFS requests. ``` -$ frostfs-http-gw -p $NEOFS_NODE -w $WALLET_PATH --address $ACCOUNT_ADDRESS +$ frostfs-http-gw -p $FROSTFS_NODE -w $WALLET_PATH --address $ACCOUNT_ADDRESS ``` Example: ``` @@ -206,7 +206,7 @@ Steps to start using name resolving: 1. Enable NNS resolving in config (`rpc_endpoint` must be a valid neo rpc node, see [configs](./config) for other examples): ```yaml -rpc_endpoint: http://morph-chain.neofs.devenv:30333 +rpc_endpoint: http://morph-chain.frostfs.devenv:30333 resolve_order: - nns ``` @@ -216,13 +216,13 @@ you can check if your container (e.g. with `container-name` name) is registered ```shell $ curl -s --data '{"id":1,"jsonrpc":"2.0","method":"getcontractstate","params":[1]}' \ - http://morph-chain.neofs.devenv:30333 | jq -r '.result.hash' + http://morph-chain.frostfs.devenv:30333 | jq -r '.result.hash' 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 $ docker exec -it morph_chain neo-go \ contract testinvokefunction \ - -r http://morph-chain.neofs.devenv:30333 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 \ + -r http://morph-chain.frostfs.devenv:30333 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 \ resolve string:container-name.container int:16 \ | jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \ | base64 -d && echo @@ -240,7 +240,7 @@ $ curl http://localhost:8082/get_by_attribute/container-name/FileName/object-nam You can create a container via [frostfs-cli](https://github.com/TrueCloudLab/frostfs-node/releases): ``` -$ frostfs-cli -r $NEOFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL +$ frostfs-cli -r $FROSTFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL ``` where `$WALLET` is a path to user wallet, `$ACL` -- hex encoded basic ACL value or keywords 'private, 'public-read', 'public-read-write' and @@ -259,7 +259,7 @@ the file `wallets/wallet.key`. To create a file via [frostfs-cli](https://github.com/TrueCloudLab/frostfs-node/releases), run a command below: ``` -$ frostfs-cli -r $NEOFS_NODE -k $KEY object put --file $FILENAME --cid $CID +$ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID ``` where `$KEY` -- the key, please read the information [above](#create-a-container), diff --git a/app.go b/app.go index 80c36f0..3328e54 100644 --- a/app.go +++ b/app.go @@ -123,9 +123,9 @@ func newApp(ctx context.Context, opt ...Option) App { a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - key, err = getNeoFSKey(a) + key, err = getFrostFSKey(a) if err != nil { - a.log.Fatal("failed to get neofs credentials", zap.Error(err)) + a.log.Fatal("failed to get frostfs credentials", zap.Error(err)) } var owner user.ID @@ -194,7 +194,7 @@ func (a *app) initResolver() { func (a *app) getResolverConfig() ([]string, *resolver.Config) { resolveCfg := &resolver.Config{ - NeoFS: resolver.NewNeoFSResolver(a.pool), + FrostFS: resolver.NewFrostFSResolver(a.pool), RPCAddress: a.cfg.GetString(cfgRPCEndpoint), } @@ -266,7 +266,7 @@ func remove(list []string, element string) []string { return list } -func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) { +func getFrostFSKey(a *app) (*ecdsa.PrivateKey, error) { walletPath := a.cfg.GetString(cfgWalletPath) if len(walletPath) == 0 { diff --git a/config/config.env b/config/config.env index fbf7327..4f13bb0 100644 --- a/config/config.env +++ b/config/config.env @@ -27,13 +27,13 @@ HTTP_GW_SERVER_1_TLS_CERT_FILE=/path/to/tls/cert HTTP_GW_SERVER_1_TLS_KEY_FILE=/path/to/tls/key # Nodes configuration. -# This configuration make the gateway use the first node (grpc://s01.neofs.devenv:8080) -# while it's healthy. Otherwise, the gateway use the second node (grpc://s01.neofs.devenv:8080) +# This configuration make the gateway use the first node (grpc://s01.frostfs.devenv:8080) +# while it's healthy. Otherwise, the gateway use the second node (grpc://s01.frostfs.devenv:8080) # for 10% of requests and the third node for 90% of requests. # Peer 1. # Endpoint. -HTTP_GW_PEERS_0_ADDRESS=grpc://s01.neofs.devenv:8080 +HTTP_GW_PEERS_0_ADDRESS=grpc://s01.frostfs.devenv:8080 # Until nodes with the same priority level are healthy # nodes with other priority are not used. # The lower the value, the higher the priority. @@ -41,11 +41,11 @@ HTTP_GW_PEERS_0_PRIORITY=1 # Load distribution proportion for nodes with the same priority. HTTP_GW_PEERS_0_WEIGHT=1 # Peer 2. -HTTP_GW_PEERS_1_ADDRESS=grpc://s02.neofs.devenv:8080 +HTTP_GW_PEERS_1_ADDRESS=grpc://s02.frostfs.devenv:8080 HTTP_GW_PEERS_1_PRIORITY=2 HTTP_GW_PEERS_1_WEIGHT=1 # Peer 3. -HTTP_GW_PEERS_2_ADDRESS=grpc://s03.neofs.devenv:8080 +HTTP_GW_PEERS_2_ADDRESS=grpc://s03.frostfs.devenv:8080 HTTP_GW_PEERS_2_PRIORITY=2 HTTP_GW_PEERS_2_WEIGHT=9 @@ -72,7 +72,7 @@ HTTP_GW_STREAM_REQUEST_BODY=true HTTP_GW_MAX_REQUEST_BODY_SIZE=4194304 # RPC endpoint to be able to use nns container resolving. -HTTP_GW_RPC_ENDPOINT=http://morph-chain.neofs.devenv:30333 +HTTP_GW_RPC_ENDPOINT=http://morph-chain.frostfs.devenv:30333 # The order in which resolvers are used to find an container id by name. HTTP_GW_RESOLVE_ORDER="nns dns" diff --git a/config/config.yaml b/config/config.yaml index fc2fada..71bab50 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -26,13 +26,13 @@ server: key_file: /path/to/key # Nodes configuration. -# This configuration make the gateway use the first node (grpc://s01.neofs.devenv:8080) -# while it's healthy. Otherwise, the gateway use the second node (grpc://s01.neofs.devenv:8080) +# This configuration make the gateway use the first node (grpc://s01.frostfs.devenv:8080) +# while it's healthy. Otherwise, the gateway use the second node (grpc://s01.frostfs.devenv:8080) # for 10% of requests and the third node for 90% of requests. peers: 0: # Endpoint. - address: grpc://s01.neofs.devenv:8080 + address: grpc://s01.frostfs.devenv:8080 # Until nodes with the same priority level are healthy # nodes with other priority are not used. @@ -42,11 +42,11 @@ peers: # Load distribution proportion for nodes with the same priority. weight: 1 1: - address: grpc://s02.neofs.devenv:8080 + address: grpc://s02.frostfs.devenv:8080 priority: 2 weight: 1 2: - address: grpc://s03.neofs.devenv:8080 + address: grpc://s03.frostfs.devenv:8080 priority: 2 weight: 9 @@ -80,7 +80,7 @@ web: max_request_body_size: 4194304 # RPC endpoint to be able to use nns container resolving. -rpc_endpoint: http://morph-chain.neofs.devenv:30333 +rpc_endpoint: http://morph-chain.frostfs.devenv:30333 # The order in which resolvers are used to find an container id by name. resolve_order: - nns diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 2f65670..d955d2a 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -57,7 +57,7 @@ $ cat http.log # General section ```yaml -rpc_endpoint: http://morph-chain.neofs.devenv:30333 +rpc_endpoint: http://morph-chain.frostfs.devenv:30333 resolve_order: - nns - dns @@ -98,23 +98,23 @@ wallet: ```yaml # Nodes configuration -# This configuration makes the gateway use the first node (node1.neofs:8080) -# while it's healthy. Otherwise, gateway uses the second node (node2.neofs:8080) -# for 10% of requests and the third node (node3.neofs:8080) for 90% of requests. +# This configuration makes the gateway use the first node (node1.frostfs:8080) +# while it's healthy. Otherwise, gateway uses the second node (node2.frostfs:8080) +# for 10% of requests and the third node (node3.frostfs:8080) for 90% of requests. # Until nodes with the same priority level are healthy # nodes with other priority are not used. # The lower the value, the higher the priority. peers: 0: - address: node1.neofs:8080 + address: node1.frostfs:8080 priority: 1 weight: 1 1: - address: node2.neofs:8080 + address: node2.frostfs:8080 priority: 2 weight: 0.1 2: - address: node3.neofs:8080 + address: node3.frostfs:8080 priority: 2 weight: 0.9 ``` diff --git a/downloader/download.go b/downloader/download.go index f34ab62..b3881a6 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -111,7 +111,7 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) { rObj, err := clnt.GetObject(r.appCtx, prm) if err != nil { - r.handleNeoFSErr(err, start) + r.handleFrostFSErr(err, start) return } @@ -224,7 +224,7 @@ func bearerToken(ctx context.Context) *bearer.Token { return nil } -func (r *request) handleNeoFSErr(err error, start time.Time) { +func (r *request) handleFrostFSErr(err error, start time.Time) { r.log.Error( "could not receive object", zap.Stringer("elapsed", time.Since(start)), @@ -500,7 +500,7 @@ func (d *Downloader) zipObject(zipWriter *zip.Writer, addr oid.Address, btoken * resGet, err := d.pool.GetObject(d.appCtx, prm) if err != nil { - return fmt.Errorf("get NeoFS object: %v", err) + return fmt.Errorf("get FrostFS object: %v", err) } objWriter, err := d.addObjectToZip(zipWriter, &resGet.Header) diff --git a/downloader/head.go b/downloader/head.go index 2487b23..3fd243e 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -44,7 +44,7 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { obj, err := clnt.HeadObject(r.appCtx, prm) if err != nil { - r.handleNeoFSErr(err, start) + r.handleFrostFSErr(err, start) return } @@ -94,7 +94,7 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { return &resObj, nil }) if err != nil && err != io.EOF { - r.handleNeoFSErr(err, start) + r.handleFrostFSErr(err, start) return } } diff --git a/metrics/metrics.go b/metrics/metrics.go index 84b7114..da57d64 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -10,7 +10,7 @@ import ( ) const ( - namespace = "neofs_http_gw" + namespace = "frostfs_http_gw" stateSubsystem = "state" poolSubsystem = "pool" diff --git a/resolver/frostfs.go b/resolver/frostfs.go new file mode 100644 index 0000000..0a1f255 --- /dev/null +++ b/resolver/frostfs.go @@ -0,0 +1,35 @@ +package resolver + +import ( + "context" + "errors" + "fmt" + + "github.com/TrueCloudLab/frostfs-sdk-go/pool" +) + +// FrostFSResolver represents virtual connection to the FrostFS network. +// It implements resolver.FrostFS. +type FrostFSResolver struct { + pool *pool.Pool +} + +// NewFrostFSResolver creates new FrostFSResolver using provided pool.Pool. +func NewFrostFSResolver(p *pool.Pool) *FrostFSResolver { + return &FrostFSResolver{pool: p} +} + +// SystemDNS implements resolver.FrostFS interface method. +func (x *FrostFSResolver) SystemDNS(ctx context.Context) (string, error) { + networkInfo, err := x.pool.NetworkInfo(ctx) + if err != nil { + return "", fmt.Errorf("read network info via client: %w", err) + } + + domain := networkInfo.RawNetworkParameter("SystemDNS") + if domain == nil { + return "", errors.New("system DNS parameter not found or empty") + } + + return string(domain), nil +} diff --git a/resolver/neofs.go b/resolver/neofs.go deleted file mode 100644 index 308a060..0000000 --- a/resolver/neofs.go +++ /dev/null @@ -1,35 +0,0 @@ -package resolver - -import ( - "context" - "errors" - "fmt" - - "github.com/TrueCloudLab/frostfs-sdk-go/pool" -) - -// NeoFSResolver represents virtual connection to the NeoFS network. -// It implements resolver.NeoFS. -type NeoFSResolver struct { - pool *pool.Pool -} - -// NewNeoFSResolver creates new NeoFSResolver using provided pool.Pool. -func NewNeoFSResolver(p *pool.Pool) *NeoFSResolver { - return &NeoFSResolver{pool: p} -} - -// SystemDNS implements resolver.NeoFS interface method. -func (x *NeoFSResolver) SystemDNS(ctx context.Context) (string, error) { - networkInfo, err := x.pool.NetworkInfo(ctx) - if err != nil { - return "", fmt.Errorf("read network info via client: %w", err) - } - - domain := networkInfo.RawNetworkParameter("SystemDNS") - if domain == nil { - return "", errors.New("system DNS parameter not found or empty") - } - - return string(domain), nil -} diff --git a/resolver/resolver.go b/resolver/resolver.go index 686a147..24f8835 100644 --- a/resolver/resolver.go +++ b/resolver/resolver.go @@ -19,9 +19,9 @@ const ( // ErrNoResolvers returns when trying to resolve container without any resolver. var ErrNoResolvers = errors.New("no resolvers") -// NeoFS represents virtual connection to the NeoFS network. -type NeoFS interface { - // SystemDNS reads system DNS network parameters of the NeoFS. +// FrostFS represents virtual connection to the FrostFS network. +type FrostFS interface { + // SystemDNS reads system DNS network parameters of the FrostFS. // // Returns exactly on non-zero value. Returns any error encountered // which prevented the parameter to be read. @@ -29,7 +29,7 @@ type NeoFS interface { } type Config struct { - NeoFS NeoFS + FrostFS FrostFS RPCAddress string } @@ -135,7 +135,7 @@ func (r *ContainerResolver) equals(resolverNames []string) bool { func newResolver(name string, cfg *Config) (*Resolver, error) { switch name { case DNSResolver: - return NewDNSResolver(cfg.NeoFS) + return NewDNSResolver(cfg.FrostFS) case NNSResolver: return NewNNSResolver(cfg.RPCAddress) default: @@ -143,17 +143,17 @@ func newResolver(name string, cfg *Config) (*Resolver, error) { } } -func NewDNSResolver(neoFS NeoFS) (*Resolver, error) { - if neoFS == nil { +func NewDNSResolver(frostFS FrostFS) (*Resolver, error) { + if frostFS == nil { return nil, fmt.Errorf("pool must not be nil for DNS resolver") } var dns ns.DNS resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) { - domain, err := neoFS.SystemDNS(ctx) + domain, err := frostFS.SystemDNS(ctx) if err != nil { - return nil, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err) + return nil, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err) } domain = name + "." + domain diff --git a/uploader/filter.go b/uploader/filter.go index 0ee5b66..2942f6d 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -13,7 +13,7 @@ import ( "go.uber.org/zap" ) -var neofsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")} +var frostfsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")} func systemTranslator(key, prefix []byte) []byte { // replace the specified prefix with `__NEOFS__` @@ -46,7 +46,7 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st clearKey := bytes.TrimPrefix(key, prefix) // checks that it's a system NeoFS header - for _, system := range neofsAttributeHeaderPrefixes { + for _, system := range frostfsAttributeHeaderPrefixes { if bytes.HasPrefix(clearKey, system) { clearKey = systemTranslator(clearKey, system) break diff --git a/uploader/upload.go b/uploader/upload.go index c60beef..9c881de 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -182,8 +182,8 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { } if idObj, err = u.pool.PutObject(u.appCtx, prm); err != nil { - log.Error("could not store file in neofs", zap.Error(err)) - response.Error(c, "could not store file in neofs: "+err.Error(), fasthttp.StatusBadRequest) + log.Error("could not store file in frostfs", zap.Error(err)) + response.Error(c, "could not store file in frostfs: "+err.Error(), fasthttp.StatusBadRequest) return } From 6abd500b11b40d699315eec95cd7e69fc430a79b Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 20 Dec 2022 18:30:13 +0300 Subject: [PATCH 348/548] [#2] Drop logo Signed-off-by: Denis Kirillov --- .github/logo.svg | 129 ----------------------------------------------- README.md | 3 -- 2 files changed, 132 deletions(-) delete mode 100644 .github/logo.svg diff --git a/.github/logo.svg b/.github/logo.svg deleted file mode 100644 index b4da076..0000000 --- a/.github/logo.svg +++ /dev/null @@ -1,129 +0,0 @@ - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/README.md b/README.md index e336a6d..6d3f85c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,3 @@ -

-FrostFS -

FrostFS is a decentralized distributed object storage integrated with the NEO Blockchain.

From 72734ab48630ec7410dc8d1c046348ab1dfb66f2 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 19 Dec 2022 11:00:44 +0300 Subject: [PATCH 349/548] Release v0.26.0 Signed-off-by: Alex Vanin --- CHANGELOG.md | 24 +++++++++++++++++++----- VERSION | 2 +- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ebef4a..51016ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,20 +4,22 @@ This document outlines major changes between releases. ## [Unreleased] +## [0.26.0] - 2022-12-28 + ### Fixed -- Download zip archive when `FilePath` is invalid (#222) -- Only one peer must be healthy to init pool (#233) +- ENV config example (#236) ### Added - Support the `Date` header on upload (#214) +- Available routes specification (#216) +- Mention caching strategy in docs (#215) - Add error response on attribute duplicates (#221) -- Timeout for individual operations in streaming RPC (#234) - Multiple server listeners (#228) ### Removed - Deprecated linters (#239) -### Updating from v0.25.0 +### Updating from v0.25.1 Make sure your configuration is valid: If you configure application using environment variables change: @@ -30,6 +32,16 @@ If you configure application using `.yaml` file change: * `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`) * `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`) +## [0.25.1] - 2022-11-30 + +### Fixed +- Download zip archive when `FilePath` is invalid (#222) +- Only one peer must be healthy to init pool (#233) + +### Added +- Debian packaging (#223) +- Timeout for individual operations in streaming RPC (#234) + ## [0.25.0] - 2022-10-31 ### Added @@ -262,4 +274,6 @@ releases. [0.23.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.22.0...v0.23.0 [0.24.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.23.0...v0.24.0 [0.25.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.24.0...v0.25.0 -[Unreleased]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.0...master +[0.25.1]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.0...v0.25.1 +[0.26.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.1...v0.26.0 +[Unreleased]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.26.0...master diff --git a/VERSION b/VERSION index 5bd9725..eaf8bae 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.25.0 +v0.26.0 From 4c30ff6638068967d0da32a224ef59ff305b92d6 Mon Sep 17 00:00:00 2001 From: Stanislav Bogatyrev Date: Sun, 8 Jan 2023 10:37:07 +0300 Subject: [PATCH 350/548] Change logo Signed-off-by: Stanislav Bogatyrev --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 6d3f85c..d27d4e8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +

+FrostFS logo +

FrostFS is a decentralized distributed object storage integrated with the NEO Blockchain.

From 913644d64a29caf159b52b8e2e368f3693d3787d Mon Sep 17 00:00:00 2001 From: Stanislav Bogatyrev Date: Mon, 9 Jan 2023 11:05:04 +0300 Subject: [PATCH 351/548] Change logo Signed-off-by: Stanislav Bogatyrev --- .github/logo.svg | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/logo.svg diff --git a/.github/logo.svg b/.github/logo.svg new file mode 100644 index 0000000..148c359 --- /dev/null +++ b/.github/logo.svg @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 7df26d918163cdf2d41c725016fda6452aacebdf Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 9 Jan 2023 11:33:41 +0300 Subject: [PATCH 352/548] [#6] Update SDK Signed-off-by: Denis Kirillov --- go.mod | 32 ++++++++++++------------- go.sum | 74 +++++++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 68 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index 7d9c3be..8920b96 100644 --- a/go.mod +++ b/go.mod @@ -4,17 +4,17 @@ go 1.17 require ( github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68 - github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221214065929-4c779423f556 + github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221230104050-dd88a5c5e0cc github.com/fasthttp/router v1.4.1 - github.com/nspcc-dev/neo-go v0.99.4 + github.com/nspcc-dev/neo-go v0.100.1 github.com/prometheus/client_golang v1.13.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 github.com/testcontainers/testcontainers-go v0.13.0 github.com/valyala/fasthttp v1.34.0 go.uber.org/atomic v1.10.0 - go.uber.org/zap v1.23.0 + go.uber.org/zap v1.24.0 ) require ( @@ -27,16 +27,15 @@ require ( github.com/TrueCloudLab/rfc6979 v0.3.0 // indirect github.com/TrueCloudLab/tzhash v1.7.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/btcsuite/btcd v0.22.0-beta // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/containerd v1.6.2 // indirect - github.com/coreos/go-semver v0.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/docker v20.10.14+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect @@ -48,7 +47,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect - github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/klauspost/compress v1.15.0 // indirect github.com/magiconair/properties v1.8.6 // indirect @@ -60,7 +59,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262 // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect @@ -83,13 +82,14 @@ require ( github.com/urfave/cli v1.22.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.opencensus.io v0.23.0 // indirect - go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect - golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect - golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.7 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.4.0 // indirect + golang.org/x/exp v0.0.0-20221227203929-1b447090c38c // indirect + golang.org/x/net v0.3.0 // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/term v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect google.golang.org/grpc v1.48.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index 3e8348f..09c6e09 100644 --- a/go.sum +++ b/go.sum @@ -103,8 +103,8 @@ github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42 h1:P github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42/go.mod h1:qmf648elr+FWBZH3hqND8KVrXMnqu/e0z48k+sX8C2s= github.com/TrueCloudLab/frostfs-crypto v0.5.0 h1:ZoLjixSkQv3j1EwZ1WJzMEJY2NR+9nO4Pd8WSyM/RRI= github.com/TrueCloudLab/frostfs-crypto v0.5.0/go.mod h1:775MUewpH8AWpXrimAG2NYWOXB6lpKOI5kqgu+eI5zs= -github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221214065929-4c779423f556 h1:Cc1jjYxKPfyw7TIJh3Bje7m8DOSn2dx+2zmr0yusWGw= -github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221214065929-4c779423f556/go.mod h1:4ZiG4jNLzrqeJbmZUrPI7wDZhQVPaf0zEIWa/eBsqBg= +github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221230104050-dd88a5c5e0cc h1:X6PCmTlUqHLhM3PaRJkVEvu23fDPx0cZWHTK2RxfOZY= +github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221230104050-dd88a5c5e0cc/go.mod h1:J5Gv7HtN9wJ5vKMy+vOT9/D6ixPMNl2ureKNNq8dask= github.com/TrueCloudLab/hrw v1.1.0 h1:2U69PpUX1UtMWgh/RAg6D8mQW+/WsxbLNE+19EUhLhY= github.com/TrueCloudLab/hrw v1.1.0/go.mod h1:Pzi8Hy3qx12cew+ajVxgbtDVM4sRG9/gJnJLcL/yRyY= github.com/TrueCloudLab/rfc6979 v0.3.0 h1:0SYMAfQWh/TjnofqYQHy+s3rmQ5gi0fvOaDbqd60/Ic= @@ -129,9 +129,10 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1 h1:zFRi26YWd7NIorBXe8UkevRl0dIvk/AnXHWaAiZG+Yk= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= -github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 h1:npHgfD4Tl2WJS3AJaMUi5ynGDPUBfkg3U3fCzDyXZ+4= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -156,7 +157,6 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -320,7 +320,6 @@ github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmeka github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -348,6 +347,9 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -599,8 +601,8 @@ github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= -github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= +github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -753,16 +755,19 @@ github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxx github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/dbft v0.0.0-20220902113116-58a5e763e647/go.mod h1:g9xisXmX9NP9MjioaTe862n9SlZTrP+6PVUWLBYOr98= +github.com/nspcc-dev/dbft v0.0.0-20221020093431-31c1bbdc74f2/go.mod h1:g9xisXmX9NP9MjioaTe862n9SlZTrP+6PVUWLBYOr98= github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 h1:n4ZaFCKt1pQJd7PXoMJabZWK9ejjbLOVrkl/lOUmshg= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= -github.com/nspcc-dev/neo-go v0.99.4 h1:8Y+SdRxksC72a4PNkcGCh/aaQinh9Gu+c5LilbcsXOI= github.com/nspcc-dev/neo-go v0.99.4/go.mod h1:mKTolfRUfKjFso5HPvGSQtUZc70n0VKBMs16eGuC5gA= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262 h1:UTmSLZw5OpD/JPE1B5Vf98GF0zu2/Hsqq1lGLtStTUE= +github.com/nspcc-dev/neo-go v0.100.1 h1:yugxbQRdzM+ObVa5mtr9/n4rYjxSIrryne8MVr9NBwU= +github.com/nspcc-dev/neo-go v0.100.1/go.mod h1:Nnp7F4e9IBccsgtCeLtUWV+0T6gk1PtP5HRtA13hUfc= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb h1:GFxfkpXEYAbMIr69JpKOsQWeLOaGrd49HNAor8uDW+A= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= @@ -966,6 +971,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -974,8 +980,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1029,6 +1036,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= @@ -1089,13 +1097,13 @@ go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= -go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1118,8 +1126,11 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1130,6 +1141,9 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/exp v0.0.0-20221227203929-1b447090c38c h1:Govq2W3bnHJimHT2ium65kXcI7ZzTniZHcFATnLJM0Q= +golang.org/x/exp v0.0.0-20221227203929-1b447090c38c/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1156,6 +1170,9 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1220,8 +1237,10 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1248,8 +1267,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1358,15 +1378,20 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1375,8 +1400,10 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1453,6 +1480,9 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 148b1aa7f5edc2b3361d0b01f29d6f1ae1ecd329 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 18 Jan 2023 12:44:44 +0300 Subject: [PATCH 353/548] [TrueCloudLab#7] Require only one healthy server Signed-off-by: Denis Kirillov --- app.go | 50 +++++++++++++++++++++++++++++++++++++------------- server.go | 10 ++++------ 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/app.go b/app.go index 3328e54..b43b144 100644 --- a/app.go +++ b/app.go @@ -485,33 +485,57 @@ func (a *app) AppParams() *utils.AppParams { func (a *app) initServers(ctx context.Context) { serversInfo := fetchServers(a.cfg) - a.servers = make([]Server, len(serversInfo)) - for i, serverInfo := range serversInfo { - a.log.Info("added server", + a.servers = make([]Server, 0, len(serversInfo)) + for _, serverInfo := range serversInfo { + fields := []zap.Field{ zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled), - zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile)) - a.servers[i] = newServer(ctx, serverInfo, a.log) + zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile), + } + srv, err := newServer(ctx, serverInfo) + if err != nil { + a.log.Warn("failed to add server", append(fields, zap.Error(err))...) + continue + } + + a.servers = append(a.servers, srv) + a.log.Info("add server", fields...) + } + + if len(a.servers) == 0 { + a.log.Fatal("no healthy servers") } } func (a *app) updateServers() error { serversInfo := fetchServers(a.cfg) - if len(serversInfo) != len(a.servers) { - return fmt.Errorf("invalid servers configuration: length mismatch: old '%d', new '%d", len(a.servers), len(serversInfo)) - } - - for i, serverInfo := range serversInfo { - if serverInfo.Address != a.servers[i].Address() { - return fmt.Errorf("invalid servers configuration: addresses mismatch: old '%s', new '%s", a.servers[i].Address(), serverInfo.Address) + var found bool + for _, serverInfo := range serversInfo { + index := a.serverIndex(serverInfo.Address) + if index == -1 { + continue } if serverInfo.TLS.Enabled { - if err := a.servers[i].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { + if err := a.servers[index].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { return fmt.Errorf("failed to update tls certs: %w", err) } } + found = true + } + + if !found { + return fmt.Errorf("invalid servers configuration: no known server found") } return nil } + +func (a *app) serverIndex(address string) int { + for i := range a.servers { + if a.servers[i].Address() == address { + return i + } + } + return -1 +} diff --git a/server.go b/server.go index 3843081..c5852d8 100644 --- a/server.go +++ b/server.go @@ -7,8 +7,6 @@ import ( "fmt" "net" "sync" - - "go.uber.org/zap" ) type ( @@ -57,11 +55,11 @@ func (s *server) UpdateCert(certFile, keyFile string) error { return s.tlsProvider.UpdateCert(certFile, keyFile) } -func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) *server { +func newServer(ctx context.Context, serverInfo ServerInfo) (*server, error) { var lic net.ListenConfig ln, err := lic.Listen(ctx, "tcp", serverInfo.Address) if err != nil { - logger.Fatal("could not prepare listener", zap.String("address", serverInfo.Address), zap.Error(err)) + return nil, fmt.Errorf("could not prepare listener: %w", err) } tlsProvider := &certProvider{ @@ -70,7 +68,7 @@ func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) * if serverInfo.TLS.Enabled { if err = tlsProvider.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { - logger.Fatal("failed to update cert", zap.Error(err)) + return nil, fmt.Errorf("failed to update cert: %w", err) } ln = tls.NewListener(ln, &tls.Config{ @@ -82,7 +80,7 @@ func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) * address: serverInfo.Address, listener: ln, tlsProvider: tlsProvider, - } + }, nil } func (p *certProvider) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { From 6909ef5382aceb618033c1eeef0e05967222902c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 18 Jan 2023 12:56:14 +0300 Subject: [PATCH 354/548] [TrueCloudLab#7] Fix tests Signed-off-by: Denis Kirillov --- integration_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/integration_test.go b/integration_test.go index 3d6f14b..e1a53f5 100644 --- a/integration_test.go +++ b/integration_test.go @@ -44,7 +44,6 @@ func TestIntegration(t *testing.T) { rootCtx := context.Background() aioImage := "nspccdev/neofs-aio-testcontainer:" versions := []string{ - "0.27.5", "0.28.1", "0.29.0", "0.30.0", From 361acacf0771303c56f96009cb2ece6a09a2cb7b Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 25 Jan 2023 10:00:02 +0300 Subject: [PATCH 355/548] [TrueCloudLab#9] Update go version to 1.18 Signed-off-by: Denis Kirillov --- .github/workflows/tests.yml | 2 +- go.mod | 2 +- go.sum | 165 ------------------------------------ 3 files changed, 2 insertions(+), 167 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cf1d462..e4c210f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - go_versions: [ '1.17', '1.18', '1.19' ] + go_versions: [ '1.18', '1.19' ] fail-fast: false steps: - uses: actions/checkout@v2 diff --git a/go.mod b/go.mod index 8920b96..870531f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/TrueCloudLab/frostfs-http-gw -go 1.17 +go 1.18 require ( github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68 diff --git a/go.sum b/go.sum index 09c6e09..8aca20f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,4 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -39,24 +38,19 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -72,7 +66,6 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= @@ -82,7 +75,6 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3 github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= -github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= github.com/Microsoft/hcsshim v0.9.2 h1:wB06W5aYFfUB3IvootYAY2WnOmIdgPGfqSI6tufQNnY= @@ -90,7 +82,6 @@ github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfy github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -122,14 +113,12 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= -github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1 h1:zFRi26YWd7NIorBXe8UkevRl0dIvk/AnXHWaAiZG+Yk= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 h1:npHgfD4Tl2WJS3AJaMUi5ynGDPUBfkg3U3fCzDyXZ+4= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= @@ -139,7 +128,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -150,7 +138,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -169,7 +156,6 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= @@ -178,9 +164,6 @@ github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -209,9 +192,6 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= @@ -249,7 +229,6 @@ github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09Zvgq github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= github.com/containerd/containerd v1.6.2 h1:pcaPUGbYW8kBw6OgIZwIVIeEhdWVrBzsoCfVJ5BjrLU= github.com/containerd/containerd v1.6.2/go.mod h1:sidY30/InSE1j2vdD1ihtKoJz+lWdaXMdiAeIupaf+s= @@ -261,7 +240,6 @@ github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/continuity v0.2.2 h1:QSqfxcn8c+12slxwu00AtzXrsami0MJb/MQs9lOLHLA= -github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= @@ -270,8 +248,6 @@ github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1S github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= -github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= -github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= @@ -281,7 +257,6 @@ github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= -github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= @@ -304,20 +279,15 @@ github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNR github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -396,16 +366,13 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fasthttp/router v1.4.1 h1:3xPUO+hy/HAkgGDSd5sX5w18cyGDIFbC7vip8KwPDk8= github.com/fasthttp/router v1.4.1/go.mod h1:4P0Kq4C882tA2evBKDW7De7hGfWmvV8FN+zqt8Lu49Q= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -413,7 +380,6 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -430,33 +396,23 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= @@ -516,7 +472,6 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -534,7 +489,6 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -560,8 +514,6 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= @@ -576,7 +528,6 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -585,13 +536,11 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -619,9 +568,7 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= -github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -629,7 +576,6 @@ github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -671,7 +617,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -679,7 +624,6 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -694,7 +638,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -714,7 +657,6 @@ github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxd github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= github.com/moby/sys/mount v0.3.2 h1:uq/CiGDZPvr+c85RYHtKIUORFbmavBUyWH3E1NEyjqI= github.com/moby/sys/mount v0.3.2/go.mod h1:iN27Ec0LtJ0Mx/++rE6t6mTdbbEEZd+oKfAHP1y6vHs= @@ -723,11 +665,8 @@ github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2J github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/mountinfo v0.6.1 h1:+H/KnGEAGRpTrEAqNVQ2AM3SiwMgJUt/TXj+Z8cmCIc= github.com/moby/sys/mountinfo v0.6.1/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= -github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -755,7 +694,6 @@ github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxx github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/dbft v0.0.0-20220902113116-58a5e763e647/go.mod h1:g9xisXmX9NP9MjioaTe862n9SlZTrP+6PVUWLBYOr98= -github.com/nspcc-dev/dbft v0.0.0-20221020093431-31c1bbdc74f2/go.mod h1:g9xisXmX9NP9MjioaTe862n9SlZTrP+6PVUWLBYOr98= github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 h1:n4ZaFCKt1pQJd7PXoMJabZWK9ejjbLOVrkl/lOUmshg= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= @@ -780,7 +718,6 @@ github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452 github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -793,9 +730,7 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -807,7 +742,6 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -816,7 +750,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -825,7 +758,6 @@ github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= -github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runc v1.1.1 h1:PJ9DSs2sVwE0iVr++pAHE6QkS9tzcVWozlPifdwMgrU= github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -839,7 +771,6 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= @@ -882,7 +813,6 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= @@ -911,11 +841,9 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 h1:N3Af8f13ooDKcIhsmFT7Z05CStZWu4C7Md0uDEy4q6o= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873/go.mod h1:dmPawKuiAeG/aFYVs2i+Dyosoo7FNcm+Pi8iK6ZUrX8= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= @@ -936,7 +864,6 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -950,7 +877,6 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -961,11 +887,9 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -997,8 +921,6 @@ github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBN github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -1018,11 +940,9 @@ github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1036,7 +956,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= @@ -1051,10 +970,6 @@ go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3C go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -1064,27 +979,7 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -1092,7 +987,6 @@ go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1121,14 +1015,10 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1141,7 +1031,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/exp v0.0.0-20221227203929-1b447090c38c h1:Govq2W3bnHJimHT2ium65kXcI7ZzTniZHcFATnLJM0Q= golang.org/x/exp v0.0.0-20221227203929-1b447090c38c/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -1170,9 +1059,6 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1218,27 +1104,20 @@ golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1254,7 +1133,6 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1336,7 +1214,6 @@ golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1349,7 +1226,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1360,16 +1236,12 @@ golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1379,17 +1251,11 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1401,7 +1267,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1410,9 +1275,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1472,7 +1335,6 @@ golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -1480,9 +1342,6 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1541,7 +1400,6 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -1553,7 +1411,6 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1565,8 +1422,6 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 h1:myaecH64R0bIEDjNORIel4iXubqzaHU1K2z8ajBwWcM= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -1592,12 +1447,9 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w= google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -1632,7 +1484,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= @@ -1652,7 +1503,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1672,55 +1522,40 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= -k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 5a98df9d2d8ecf415022224f89be62e3937421be Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 25 Jan 2023 10:01:10 +0300 Subject: [PATCH 356/548] [TrueCloudLab#9] Update CHANGELOG.md Signed-off-by: Denis Kirillov --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51016ac..686e250 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ This document outlines major changes between releases. ## [Unreleased] +### Changed +- Update go version to 1.18 (TrueCloudLab#9) + ## [0.26.0] - 2022-12-28 ### Fixed From 38aa6db041df026f846cf582c73787c7852131da Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 25 Jan 2023 11:19:30 +0300 Subject: [PATCH 357/548] [TrueCloudLab#9] Fix tests Signed-off-by: Denis Kirillov --- integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_test.go b/integration_test.go index e1a53f5..7363e2d 100644 --- a/integration_test.go +++ b/integration_test.go @@ -44,10 +44,10 @@ func TestIntegration(t *testing.T) { rootCtx := context.Background() aioImage := "nspccdev/neofs-aio-testcontainer:" versions := []string{ - "0.28.1", "0.29.0", "0.30.0", "0.32.0", + "0.34.0", "latest", } key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") From 5f01abf3004f1851bb6900490400be511b6f5e2d Mon Sep 17 00:00:00 2001 From: Artem Tataurov Date: Wed, 25 Jan 2023 14:30:23 +0300 Subject: [PATCH 358/548] [#8] Update neo-go and viper Signed-off-by: Artem Tataurov --- CHANGELOG.md | 2 + go.mod | 31 +++++----- go.sum | 166 +++++++++++++++------------------------------------ 3 files changed, 64 insertions(+), 135 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 686e250..f06eed9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ This document outlines major changes between releases. ### Changed - Update go version to 1.18 (TrueCloudLab#9) +- Update neo-go to v0.101.0 (#8) +- Update viper to v1.15.0 (#8) ## [0.26.0] - 2022-12-28 diff --git a/go.mod b/go.mod index 870531f..e461b50 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ require ( github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68 github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221230104050-dd88a5c5e0cc github.com/fasthttp/router v1.4.1 - github.com/nspcc-dev/neo-go v0.100.1 + github.com/nspcc-dev/neo-go v0.101.0 github.com/prometheus/client_golang v1.13.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.8.1 + github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 github.com/testcontainers/testcontainers-go v0.13.0 github.com/valyala/fasthttp v1.34.0 @@ -40,7 +40,7 @@ require ( github.com/docker/docker v20.10.14+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -50,9 +50,9 @@ require ( github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/klauspost/compress v1.15.0 // indirect - github.com/magiconair/properties v1.8.6 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/sys/mount v0.3.2 // indirect github.com/moby/sys/mountinfo v0.6.1 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect @@ -64,7 +64,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.1.1 // indirect - github.com/pelletier/go-toml v1.9.3 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -74,27 +74,26 @@ require ( github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/afero v1.9.3 // indirect + github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/urfave/cli v1.22.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - go.opencensus.io v0.23.0 // indirect + go.opencensus.io v0.24.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.4.0 // indirect golang.org/x/exp v0.0.0-20221227203929-1b447090c38c // indirect - golang.org/x/net v0.3.0 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/net v0.4.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.3.0 // indirect golang.org/x/term v0.3.0 // indirect golang.org/x/text v0.5.0 // indirect - google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect - google.golang.org/grpc v1.48.0 // indirect + google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect + google.golang.org/grpc v1.52.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/ini.v1 v1.62.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8aca20f..954918d 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -16,9 +17,7 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -27,7 +26,6 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -37,6 +35,7 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -122,10 +121,7 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 h1:npHgfD4Tl2WJS3AJaMUi5ynGDPUBfkg3U3fCzDyXZ+4= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -138,7 +134,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= @@ -185,12 +180,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= @@ -359,10 +350,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -375,9 +364,11 @@ github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:r github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -461,7 +452,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -484,8 +474,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -501,8 +491,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -514,7 +503,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -532,21 +521,10 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -554,10 +532,6 @@ github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -585,7 +559,6 @@ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -618,8 +591,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -629,7 +602,6 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= @@ -642,19 +614,12 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= @@ -701,8 +666,8 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= github.com/nspcc-dev/neo-go v0.99.4/go.mod h1:mKTolfRUfKjFso5HPvGSQtUZc70n0VKBMs16eGuC5gA= -github.com/nspcc-dev/neo-go v0.100.1 h1:yugxbQRdzM+ObVa5mtr9/n4rYjxSIrryne8MVr9NBwU= -github.com/nspcc-dev/neo-go v0.100.1/go.mod h1:Nnp7F4e9IBccsgtCeLtUWV+0T6gk1PtP5HRtA13hUfc= +github.com/nspcc-dev/neo-go v0.101.0 h1:JPT2DpZqVjho34TMR59dm6uxvCFttOp02Nm8qCjpfaU= +github.com/nspcc-dev/neo-go v0.101.0/go.mod h1:Q0uWKivGc2mYgdKFmTNP49LeXwMu4x6pUzHm3OIsN2I= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb h1:GFxfkpXEYAbMIr69JpKOsQWeLOaGrd49HNAor8uDW+A= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= @@ -771,11 +736,10 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -783,10 +747,9 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -839,13 +802,11 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 h1:N3Af8f13ooDKcIhsmFT7Z05CStZWu4C7Md0uDEy4q6o= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873/go.mod h1:dmPawKuiAeG/aFYVs2i+Dyosoo7FNcm+Pi8iK6ZUrX8= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -858,22 +819,19 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= @@ -887,8 +845,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -907,8 +865,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -967,9 +925,6 @@ go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -977,8 +932,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -994,7 +949,6 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= @@ -1002,13 +956,11 @@ golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1016,8 +968,10 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= @@ -1046,7 +1000,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -1064,9 +1017,7 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1106,9 +1057,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1118,8 +1067,8 @@ golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1129,9 +1078,6 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1146,13 +1092,11 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1228,14 +1172,11 @@ golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1251,6 +1192,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1264,7 +1206,6 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= @@ -1275,7 +1216,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1300,7 +1241,6 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1337,9 +1277,9 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1366,9 +1306,6 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1416,14 +1353,10 @@ google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 h1:myaecH64R0bIEDjNORIel4iXubqzaHU1K2z8ajBwWcM= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1446,13 +1379,10 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w= -google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= +google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1466,7 +1396,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= @@ -1484,8 +1413,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -1500,7 +1429,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From ed983f8ad0223ea20ab3d47b33cc7f453f37d0e6 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 1 Feb 2023 11:13:07 +0300 Subject: [PATCH 359/548] [TrueCloudLab#11] Update SDK to renew tokens beforehand Signed-off-by: Denis Kirillov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e461b50..b7f3215 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68 - github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221230104050-dd88a5c5e0cc + github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20230130120602-cf64ddfb143c github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.0 github.com/prometheus/client_golang v1.13.0 diff --git a/go.sum b/go.sum index 954918d..2cd5eb9 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,8 @@ github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42 h1:P github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42/go.mod h1:qmf648elr+FWBZH3hqND8KVrXMnqu/e0z48k+sX8C2s= github.com/TrueCloudLab/frostfs-crypto v0.5.0 h1:ZoLjixSkQv3j1EwZ1WJzMEJY2NR+9nO4Pd8WSyM/RRI= github.com/TrueCloudLab/frostfs-crypto v0.5.0/go.mod h1:775MUewpH8AWpXrimAG2NYWOXB6lpKOI5kqgu+eI5zs= -github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221230104050-dd88a5c5e0cc h1:X6PCmTlUqHLhM3PaRJkVEvu23fDPx0cZWHTK2RxfOZY= -github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221230104050-dd88a5c5e0cc/go.mod h1:J5Gv7HtN9wJ5vKMy+vOT9/D6ixPMNl2ureKNNq8dask= +github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20230130120602-cf64ddfb143c h1:7TTjeRSWHOSmFEsblebGBN4aLeZEVVN9JixQvu2tD7Q= +github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20230130120602-cf64ddfb143c/go.mod h1:J5Gv7HtN9wJ5vKMy+vOT9/D6ixPMNl2ureKNNq8dask= github.com/TrueCloudLab/hrw v1.1.0 h1:2U69PpUX1UtMWgh/RAg6D8mQW+/WsxbLNE+19EUhLhY= github.com/TrueCloudLab/hrw v1.1.0/go.mod h1:Pzi8Hy3qx12cew+ajVxgbtDVM4sRG9/gJnJLcL/yRyY= github.com/TrueCloudLab/rfc6979 v0.3.0 h1:0SYMAfQWh/TjnofqYQHy+s3rmQ5gi0fvOaDbqd60/Ic= From 6be8d47d922d02e3949e0e71ef95be624e901e18 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 3 Feb 2023 12:46:25 +0300 Subject: [PATCH 360/548] [TrueCloudLab#12] Support multiple configs Signed-off-by: Denis Kirillov --- README.md | 17 +++++++++ app.go | 5 ++- config/config.yaml | 4 +- config/dir/pprof.yaml | 3 ++ config/dir/prometheus.yaml | 3 ++ settings.go | 76 +++++++++++++++++++++++++++++++++----- 6 files changed, 94 insertions(+), 14 deletions(-) create mode 100644 config/dir/pprof.yaml create mode 100644 config/dir/prometheus.yaml diff --git a/README.md b/README.md index d27d4e8..7d2e315 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,23 @@ $ frostfs-http-gw --config your-config.yaml See [config](./config/config.yaml) and [defaults](./docs/gate-configuration.md) for example. +#### Multiple configs + +You can use several config files when running application. It allows you to split configuration into parts. +For example, you can use separate yaml file for pprof and prometheus section in config (see [config examples](./config)). +You can either provide several files with repeating `--config` flag or provide path to the dir that contains all configs using `--config-dir` flag. +Also, you can combine these flags: + +```shell +$ frostfs-http-gw --config ./config/config.yaml --config /your/partial/config.yaml --config-dir ./config/dir +``` + +**Note:** next file in `--config` flag overwrites values from the previous one. +Files from `--config-dir` directory overwrite values from `--config` files. +So the command above run `frostfs-http-gw` to listen on `0.0.0.0:8080` address (value from `./config/config.yaml`), +applies parameters from `/your/partial/config.yaml`, +enable pprof (value from `./config/dir/pprof.yaml`) and prometheus (value from `./config/dir/prometheus.yaml`). + ## HTTP API provided This gateway intentionally provides limited feature set and doesn't try to diff --git a/app.go b/app.go index b43b144..e17aca9 100644 --- a/app.go +++ b/app.go @@ -380,14 +380,15 @@ LOOP: func (a *app) configReload() { a.log.Info("SIGHUP config reload started") - if !a.cfg.IsSet(cmdConfig) { + if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) { a.log.Warn("failed to reload config because it's missed") return } - if err := readConfig(a.cfg); err != nil { + if err := readInConfig(a.cfg); err != nil { a.log.Warn("failed to reload config", zap.Error(err)) return } + if lvl, err := getLogLevel(a.cfg); err != nil { a.log.Warn("log level won't be updated", zap.Error(err)) } else { diff --git a/config/config.yaml b/config/config.yaml index 71bab50..0a117df 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -4,10 +4,10 @@ wallet: passphrase: pwd # Passphrase to decrypt wallet. If you're using a wallet without a password, place '' here. pprof: - enabled: true # Enable pprof. + enabled: false # Enable pprof. address: localhost:8083 prometheus: - enabled: true # Enable metrics. + enabled: false # Enable metrics. address: localhost:8084 logger: diff --git a/config/dir/pprof.yaml b/config/dir/pprof.yaml new file mode 100644 index 0000000..44f93ca --- /dev/null +++ b/config/dir/pprof.yaml @@ -0,0 +1,3 @@ +pprof: + enabled: true + address: localhost:8083 diff --git a/config/dir/prometheus.yaml b/config/dir/prometheus.yaml new file mode 100644 index 0000000..f29db69 --- /dev/null +++ b/config/dir/prometheus.yaml @@ -0,0 +1,3 @@ +prometheus: + enabled: true + address: localhost:8084 diff --git a/settings.go b/settings.go index c6ee33f..cd71e4a 100644 --- a/settings.go +++ b/settings.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "path" "runtime" "sort" "strconv" @@ -84,6 +85,7 @@ const ( cmdWallet = "wallet" cmdAddress = "address" cmdConfig = "config" + cmdConfigDir = "config-dir" cmdListenAddress = "listen_address" ) @@ -114,7 +116,8 @@ func settings() *viper.Viper { flags.StringP(cmdWallet, "w", "", `path to the wallet`) flags.String(cmdAddress, "", `address of wallet account`) - flags.String(cmdConfig, "", "config path") + flags.StringArray(cmdConfig, nil, "config paths") + flags.String(cmdConfigDir, "", "config dir path") flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout") flags.Duration(cfgStreamTimeout, defaultStreamTimeout, "gRPC individual message timeout") flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout") @@ -233,10 +236,8 @@ func settings() *viper.Viper { os.Exit(0) } - if v.IsSet(cmdConfig) { - if err := readConfig(v); err != nil { - panic(err) - } + if err := readInConfig(v); err != nil { + panic(err) } if peers != nil && len(*peers) > 0 { @@ -250,17 +251,72 @@ func settings() *viper.Viper { return v } -func readConfig(v *viper.Viper) error { - cfgFileName := v.GetString(cmdConfig) - cfgFile, err := os.Open(cfgFileName) +func readInConfig(v *viper.Viper) error { + if v.IsSet(cmdConfig) { + if err := readConfig(v); err != nil { + return err + } + } + + if v.IsSet(cmdConfigDir) { + if err := readConfigDir(v); err != nil { + return err + } + } + + return nil +} + +func readConfigDir(v *viper.Viper) error { + cfgSubConfigDir := v.GetString(cmdConfigDir) + entries, err := os.ReadDir(cfgSubConfigDir) if err != nil { return err } - if err = v.ReadConfig(cfgFile); err != nil { + + for _, entry := range entries { + if entry.IsDir() { + continue + } + ext := path.Ext(entry.Name()) + if ext != ".yaml" && ext != ".yml" { + continue + } + + if err = mergeConfig(v, path.Join(cfgSubConfigDir, entry.Name())); err != nil { + return err + } + } + + return nil +} + +func readConfig(v *viper.Viper) error { + for _, fileName := range v.GetStringSlice(cmdConfig) { + if err := mergeConfig(v, fileName); err != nil { + return err + } + } + return nil +} + +func mergeConfig(v *viper.Viper, fileName string) error { + cfgFile, err := os.Open(fileName) + if err != nil { return err } - return cfgFile.Close() + defer func() { + if errClose := cfgFile.Close(); errClose != nil { + panic(errClose) + } + }() + + if err = v.MergeConfig(cfgFile); err != nil { + return err + } + + return nil } // newLogger constructs a zap.Logger instance for current application. From 93ec4c444da11e556149caf5111400b0e1958bf8 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 3 Feb 2023 12:47:18 +0300 Subject: [PATCH 361/548] [TrueCloudLab#12] Update CHANGELOG.md Signed-off-by: Denis Kirillov --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f06eed9..13239c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ This document outlines major changes between releases. ## [Unreleased] +### Added +- Multiple configs support (TrueCloudLab#12) + ### Changed - Update go version to 1.18 (TrueCloudLab#9) - Update neo-go to v0.101.0 (#8) From e02ee50d7b309911f8b050e0b73d976037bc9b63 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 7 Mar 2023 17:08:53 +0300 Subject: [PATCH 362/548] Rename package name Due to source code relocation from GitHub. Signed-off-by: Alex Vanin --- README.md | 16 +++++++--------- app.go | 16 ++++++++-------- debian/control | 4 ++-- debian/copyright | 4 ++-- downloader/download.go | 22 +++++++++++----------- downloader/head.go | 12 ++++++------ go.mod | 17 +++++++++-------- go.sum | 30 ++++++++++++++++-------------- integration_test.go | 16 ++++++++-------- metrics/metrics.go | 2 +- resolver/frostfs.go | 2 +- resolver/resolver.go | 6 +++--- settings.go | 2 +- tokens/bearer-token.go | 2 +- tokens/bearer-token_test.go | 4 ++-- uploader/filter.go | 4 ++-- uploader/filter_test.go | 4 ++-- uploader/multipart.go | 2 +- uploader/upload.go | 18 +++++++++--------- utils/params.go | 6 +++--- utils/util.go | 4 ++-- 21 files changed, 97 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 7d2e315..de81301 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,7 @@

--- -[![Report](https://goreportcard.com/badge/github.com/TrueCloudLab/frostfs-http-gw)](https://goreportcard.com/report/github.com/TrueCloudLab/frostfs-http-gw) -![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/TrueCloudLab/frostfs-http-gw?sort=semver) -![License](https://img.shields.io/github/license/TrueCloudLab/frostfs-http-gw.svg?style=popout) +[![Report](https://goreportcard.com/badge/git.frostfs.info/TrueCloudLab/frostfs-http-gw)](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-http-gw) # FrostFS HTTP Gateway @@ -20,7 +18,7 @@ See available routes in [specification](./docs/api.md). ## Installation -```go install github.com/TrueCloudLab/frostfs-http-gw``` +```go install git.frostfs.info/TrueCloudLab/frostfs-http-gw``` Or you can call `make` to build it from the cloned repository (the binary will end up in `bin/frostfs-http-gw`). To build frostfs-http-gw binary in clean docker @@ -50,7 +48,7 @@ can be done either via `-p` parameter or via `HTTP_GW_PEERS__ADDRESS` and `HTTP_GW_PEERS__WEIGHT` environment variables (the gate supports multiple FrostFS nodes with weighted load balancing). -If you launch HTTP gateway in bundle with [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env), +If you launch HTTP gateway in bundle with [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env), you can get the IP address of the node in the output of `make hosts` command (with s0*.frostfs.devenv name). @@ -228,7 +226,7 @@ resolve_order: - nns ``` -2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env) +2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env) you can check if your container (e.g. with `container-name` name) is registered in NNS: ```shell @@ -255,7 +253,7 @@ $ curl http://localhost:8082/get_by_attribute/container-name/FileName/object-nam #### Create a container -You can create a container via [frostfs-cli](https://github.com/TrueCloudLab/frostfs-node/releases): +You can create a container via [frostfs-cli](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases): ``` $ frostfs-cli -r $FROSTFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL ``` @@ -268,13 +266,13 @@ For example: $ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy "REP 3" --basic-acl public --await ``` -If you have launched nodes via [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env), +If you have launched nodes via [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env), you can get the key value from `wallets/wallet.json` or write the path to the file `wallets/wallet.key`. #### Prepare a file in a container -To create a file via [frostfs-cli](https://github.com/TrueCloudLab/frostfs-node/releases), run a command below: +To create a file via [frostfs-cli](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases), run a command below: ``` $ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID ``` diff --git a/app.go b/app.go index e17aca9..825da8e 100644 --- a/app.go +++ b/app.go @@ -11,14 +11,14 @@ import ( "sync" "syscall" - "github.com/TrueCloudLab/frostfs-http-gw/downloader" - "github.com/TrueCloudLab/frostfs-http-gw/metrics" - "github.com/TrueCloudLab/frostfs-http-gw/resolver" - "github.com/TrueCloudLab/frostfs-http-gw/response" - "github.com/TrueCloudLab/frostfs-http-gw/uploader" - "github.com/TrueCloudLab/frostfs-http-gw/utils" - "github.com/TrueCloudLab/frostfs-sdk-go/pool" - "github.com/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/downloader" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/fasthttp/router" "github.com/nspcc-dev/neo-go/cli/flags" "github.com/nspcc-dev/neo-go/cli/input" diff --git a/debian/control b/debian/control index cd18ad5..612be94 100644 --- a/debian/control +++ b/debian/control @@ -5,8 +5,8 @@ Maintainer: TrueCloudLab Build-Depends: debhelper-compat (= 13), dh-sysuser, git, devscripts Standards-Version: 4.5.1 Homepage: https://frostfs.info/ -Vcs-Git: https://github.com/TrueCloudLab/frostfs-http-gw.git -Vcs-Browser: https://github.com/TrueCloudLab/frostfs-http-gw +Vcs-Git: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw.git +Vcs-Browser: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw Package: frostfs-http-gw Architecture: any diff --git a/debian/copyright b/debian/copyright index be82996..c278a3a 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,13 +1,13 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: frostfs-http-gw Upstream-Contact: tech@frostfs.info -Source: https://github.com/TrueCloudLab/frostfs-http-gw +Source: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw Files: * Copyright: 2018-2022 NeoSPCC (@nspcc-dev), contributors of neofs-http-gw project (https://github.com/nspcc-dev/neofs-http-gw/blob/master/CREDITS.md) 2022 True Cloud Lab (@TrueCloudLab), contributors of frostfs-http-gw project - (https://github.com/TrueCloudLab/frostfs-http-gw/blob/master/CREDITS.md) + (https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/src/branch/master/CREDITS.md) License: GPL-3 diff --git a/downloader/download.go b/downloader/download.go index b3881a6..f357550 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -17,17 +17,17 @@ import ( "unicode" "unicode/utf8" - "github.com/TrueCloudLab/frostfs-http-gw/resolver" - "github.com/TrueCloudLab/frostfs-http-gw/response" - "github.com/TrueCloudLab/frostfs-http-gw/tokens" - "github.com/TrueCloudLab/frostfs-http-gw/utils" - "github.com/TrueCloudLab/frostfs-sdk-go/bearer" - "github.com/TrueCloudLab/frostfs-sdk-go/client" - "github.com/TrueCloudLab/frostfs-sdk-go/container" - cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" - "github.com/TrueCloudLab/frostfs-sdk-go/object" - oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/TrueCloudLab/frostfs-sdk-go/pool" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/atomic" "go.uber.org/zap" diff --git a/downloader/head.go b/downloader/head.go index 3fd243e..36f8b60 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -7,12 +7,12 @@ import ( "strings" "time" - "github.com/TrueCloudLab/frostfs-http-gw/response" - "github.com/TrueCloudLab/frostfs-http-gw/tokens" - "github.com/TrueCloudLab/frostfs-http-gw/utils" - "github.com/TrueCloudLab/frostfs-sdk-go/object" - oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/TrueCloudLab/frostfs-sdk-go/pool" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" ) diff --git a/go.mod b/go.mod index b7f3215..4b7319e 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,10 @@ -module github.com/TrueCloudLab/frostfs-http-gw +module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.18 require ( - github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68 - github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20230130120602-cf64ddfb143c + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.0 github.com/prometheus/client_golang v1.13.0 @@ -18,14 +18,14 @@ require ( ) require ( + git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb // indirect + git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect + git.frostfs.info/TrueCloudLab/hrw v1.2.0 // indirect + git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect + git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/hcsshim v0.9.2 // indirect - github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42 // indirect - github.com/TrueCloudLab/frostfs-crypto v0.5.0 // indirect - github.com/TrueCloudLab/hrw v1.1.0 // indirect - github.com/TrueCloudLab/rfc6979 v0.3.0 // indirect - github.com/TrueCloudLab/tzhash v1.7.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -48,6 +48,7 @@ require ( github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/klauspost/compress v1.15.0 // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/go.sum b/go.sum index 2cd5eb9..585439e 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,20 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51 h1:l4+K1hN+NuWNtlZZoV8yRRP3Uu7PifL05ukEqKcb0Ks= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51/go.mod h1:n0DxKYulu2Ar73R6OcNF34LiL/Xa+iDR7GZuaOChbLE= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= +git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= +git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599 h1:mzGX2RX8R8H/tUqrUu1TcYk4QRDBcBIWGYscPncfLOQ= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599/go.mod h1:z7zcpGY+puI5puyy5oyFbf20vWp84WtslCxcr6/kv5c= +git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= +git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= +git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= +git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= +git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= +git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -87,20 +101,6 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68 h1:mwZr15qCuIcWojIOmH6LILPohbWIkknZe9vhBRapmfQ= -github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68/go.mod h1:u3P6aL/NpAIY5IFRsJhmV+61Q3pJ3BkLENqySkf5zZQ= -github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42 h1:P/gisZxTzJ9R3tuYDaQWe0tY6m1Zea3gzdPpNYK+NP4= -github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42/go.mod h1:qmf648elr+FWBZH3hqND8KVrXMnqu/e0z48k+sX8C2s= -github.com/TrueCloudLab/frostfs-crypto v0.5.0 h1:ZoLjixSkQv3j1EwZ1WJzMEJY2NR+9nO4Pd8WSyM/RRI= -github.com/TrueCloudLab/frostfs-crypto v0.5.0/go.mod h1:775MUewpH8AWpXrimAG2NYWOXB6lpKOI5kqgu+eI5zs= -github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20230130120602-cf64ddfb143c h1:7TTjeRSWHOSmFEsblebGBN4aLeZEVVN9JixQvu2tD7Q= -github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20230130120602-cf64ddfb143c/go.mod h1:J5Gv7HtN9wJ5vKMy+vOT9/D6ixPMNl2ureKNNq8dask= -github.com/TrueCloudLab/hrw v1.1.0 h1:2U69PpUX1UtMWgh/RAg6D8mQW+/WsxbLNE+19EUhLhY= -github.com/TrueCloudLab/hrw v1.1.0/go.mod h1:Pzi8Hy3qx12cew+ajVxgbtDVM4sRG9/gJnJLcL/yRyY= -github.com/TrueCloudLab/rfc6979 v0.3.0 h1:0SYMAfQWh/TjnofqYQHy+s3rmQ5gi0fvOaDbqd60/Ic= -github.com/TrueCloudLab/rfc6979 v0.3.0/go.mod h1:qylxFXFQ/sMvpZC/8JyWp+mfzk5Zj/KDT5FAbekhobc= -github.com/TrueCloudLab/tzhash v1.7.0 h1:btGORepc7Dg+n4MxgJxv73c9eYhwSBI5HqsqUBRmJiw= -github.com/TrueCloudLab/tzhash v1.7.0/go.mod h1:gDQxqjhTqhR58Qfx0gxGtuyGAkixOukwbFGX9O6UGg4= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/ishell/v2 v2.0.2/go.mod h1:E4oTCXfo6QjoCart0QYa5m9w4S+deXs/P/9jA77A9Bs= @@ -530,6 +530,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= +github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= diff --git a/integration_test.go b/integration_test.go index 7363e2d..0587273 100644 --- a/integration_test.go +++ b/integration_test.go @@ -13,14 +13,14 @@ import ( "testing" "time" - "github.com/TrueCloudLab/frostfs-sdk-go/container" - "github.com/TrueCloudLab/frostfs-sdk-go/container/acl" - cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" - "github.com/TrueCloudLab/frostfs-sdk-go/netmap" - "github.com/TrueCloudLab/frostfs-sdk-go/object" - oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/TrueCloudLab/frostfs-sdk-go/pool" - "github.com/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/viper" "github.com/stretchr/testify/require" diff --git a/metrics/metrics.go b/metrics/metrics.go index da57d64..a184588 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -3,7 +3,7 @@ package metrics import ( "net/http" - "github.com/TrueCloudLab/frostfs-sdk-go/pool" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/zap" diff --git a/resolver/frostfs.go b/resolver/frostfs.go index 0a1f255..aa7a751 100644 --- a/resolver/frostfs.go +++ b/resolver/frostfs.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "github.com/TrueCloudLab/frostfs-sdk-go/pool" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" ) // FrostFSResolver represents virtual connection to the FrostFS network. diff --git a/resolver/resolver.go b/resolver/resolver.go index 24f8835..e6707e2 100644 --- a/resolver/resolver.go +++ b/resolver/resolver.go @@ -6,9 +6,9 @@ import ( "fmt" "sync" - "github.com/TrueCloudLab/frostfs-sdk-go/container" - cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" - "github.com/TrueCloudLab/frostfs-sdk-go/ns" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns" ) const ( diff --git a/settings.go b/settings.go index cd71e4a..8d569c0 100644 --- a/settings.go +++ b/settings.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "github.com/TrueCloudLab/frostfs-http-gw/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/valyala/fasthttp" diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index e1d46a9..6cd98ca 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -7,7 +7,7 @@ import ( "errors" "fmt" - "github.com/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "github.com/valyala/fasthttp" ) diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index 07df0f9..b03147c 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -4,8 +4,8 @@ import ( "encoding/base64" "testing" - "github.com/TrueCloudLab/frostfs-sdk-go/bearer" - "github.com/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" diff --git a/uploader/filter.go b/uploader/filter.go index 2942f6d..acab646 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -7,8 +7,8 @@ import ( "strconv" "time" - "github.com/TrueCloudLab/frostfs-api-go/v2/object" - "github.com/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "github.com/valyala/fasthttp" "go.uber.org/zap" ) diff --git a/uploader/filter_test.go b/uploader/filter_test.go index dc6f462..9cb58c1 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "github.com/TrueCloudLab/frostfs-api-go/v2/object" - "github.com/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" "go.uber.org/zap" diff --git a/uploader/multipart.go b/uploader/multipart.go index ae03cc0..fb6be22 100644 --- a/uploader/multipart.go +++ b/uploader/multipart.go @@ -3,7 +3,7 @@ package uploader import ( "io" - "github.com/TrueCloudLab/frostfs-http-gw/uploader/multipart" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader/multipart" "go.uber.org/zap" ) diff --git a/uploader/upload.go b/uploader/upload.go index 9c881de..fc904db 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -9,15 +9,15 @@ import ( "strconv" "time" - "github.com/TrueCloudLab/frostfs-http-gw/resolver" - "github.com/TrueCloudLab/frostfs-http-gw/response" - "github.com/TrueCloudLab/frostfs-http-gw/tokens" - "github.com/TrueCloudLab/frostfs-http-gw/utils" - "github.com/TrueCloudLab/frostfs-sdk-go/bearer" - "github.com/TrueCloudLab/frostfs-sdk-go/object" - oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/TrueCloudLab/frostfs-sdk-go/pool" - "github.com/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/valyala/fasthttp" "go.uber.org/atomic" "go.uber.org/zap" diff --git a/utils/params.go b/utils/params.go index 68d2921..a6fe59b 100644 --- a/utils/params.go +++ b/utils/params.go @@ -1,9 +1,9 @@ package utils import ( - "github.com/TrueCloudLab/frostfs-http-gw/resolver" - "github.com/TrueCloudLab/frostfs-sdk-go/pool" - "github.com/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "go.uber.org/zap" ) diff --git a/utils/util.go b/utils/util.go index ac6edb4..a83064c 100644 --- a/utils/util.go +++ b/utils/util.go @@ -3,8 +3,8 @@ package utils import ( "context" - "github.com/TrueCloudLab/frostfs-http-gw/resolver" - cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) // GetContainerID decode container id, if it's not a valid container id From 8f6be59e23749506b2cf383157d44f1603847667 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 13 Mar 2023 14:55:19 +0300 Subject: [PATCH 363/548] [#18] Extract error details Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + downloader/download.go | 14 +++++--------- response/utils.go | 36 +++++++++++++++++++++++++++++++++++- uploader/upload.go | 11 +++++++++-- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13239c3..dec0627 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ This document outlines major changes between releases. - Update go version to 1.18 (TrueCloudLab#9) - Update neo-go to v0.101.0 (#8) - Update viper to v1.15.0 (#8) +- Errors have become more detailed (#18) ## [0.26.0] - 2022-12-28 diff --git a/downloader/download.go b/downloader/download.go index f357550..d8f0bc2 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -225,19 +225,15 @@ func bearerToken(ctx context.Context) *bearer.Token { } func (r *request) handleFrostFSErr(err error, start time.Time) { - r.log.Error( - "could not receive object", + logFields := []zap.Field{ zap.Stringer("elapsed", time.Since(start)), zap.Error(err), - ) - - if client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err) { - response.Error(r.RequestCtx, "Not Found", fasthttp.StatusNotFound) - return } + statusCode, msg, additionalFields := response.FormErrorResponse("could not receive object", err) + logFields = append(logFields, additionalFields...) - msg := fmt.Sprintf("could not receive object: %v", err) - response.Error(r.RequestCtx, msg, fasthttp.StatusBadRequest) + r.log.Error("could not receive object", logFields...) + response.Error(r.RequestCtx, msg, statusCode) } // Downloader is a download request handler. diff --git a/response/utils.go b/response/utils.go index 8a7f383..c467d5e 100644 --- a/response/utils.go +++ b/response/utils.go @@ -1,7 +1,41 @@ package response -import "github.com/valyala/fasthttp" +import ( + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) func Error(r *fasthttp.RequestCtx, msg string, code int) { r.Error(msg+"\n", code) } + +func FormErrorResponse(message string, err error) (int, string, []zap.Field) { + var ( + msg string + statusCode int + logFields []zap.Field + ) + + st := new(sdkstatus.ObjectAccessDenied) + + switch { + case errors.As(err, &st): + statusCode = fasthttp.StatusForbidden + reason := st.Reason() + msg = fmt.Sprintf("%s: %v: %s", message, err, reason) + logFields = append(logFields, zap.String("error_detail", reason)) + case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err): + statusCode = fasthttp.StatusNotFound + msg = "NotFound" + default: + statusCode = fasthttp.StatusBadRequest + msg = fmt.Sprintf("%s: %v", message, err) + } + + return statusCode, msg, logFields +} diff --git a/uploader/upload.go b/uploader/upload.go index fc904db..d455a13 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -182,8 +182,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { } if idObj, err = u.pool.PutObject(u.appCtx, prm); err != nil { - log.Error("could not store file in frostfs", zap.Error(err)) - response.Error(c, "could not store file in frostfs: "+err.Error(), fasthttp.StatusBadRequest) + u.handlePutFrostFSErr(c, err) return } @@ -214,6 +213,14 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { c.Response.Header.SetContentType(jsonHeader) } +func (u *Uploader) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) { + statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err) + logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) + + u.log.Error("could not store file in frostfs", logFields...) + response.Error(r, msg, statusCode) +} + func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*user.ID, *bearer.Token) { if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil { issuer := bearer.ResolveIssuer(*tkn) From 53ee124b196b0ccb5bf980b6977d7141caab2040 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 22 Mar 2023 10:43:34 +0300 Subject: [PATCH 364/548] [#18] Fix typo in error message Signed-off-by: Denis Kirillov --- response/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/response/utils.go b/response/utils.go index c467d5e..f233943 100644 --- a/response/utils.go +++ b/response/utils.go @@ -31,7 +31,7 @@ func FormErrorResponse(message string, err error) (int, string, []zap.Field) { logFields = append(logFields, zap.String("error_detail", reason)) case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err): statusCode = fasthttp.StatusNotFound - msg = "NotFound" + msg = "Not Found" default: statusCode = fasthttp.StatusBadRequest msg = fmt.Sprintf("%s: %v", message, err) From e2059a8926e376b1f45a1ad56eeb08f7519c0511 Mon Sep 17 00:00:00 2001 From: Liza Date: Tue, 21 Mar 2023 18:30:48 +0300 Subject: [PATCH 365/548] [#21] Add bug label Add bug label in the bug report template Signed-off-by: Liza --- .github/ISSUE_TEMPLATE/bug_report.md | 12 ++++++------ .github/ISSUE_TEMPLATE/feature_request.md | 16 ++++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 86b9297..2861ed3 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,7 +2,7 @@ name: Bug report about: Create a report to help us improve title: '' -labels: community, triage +labels: community, triage, bug assignees: '' --- @@ -18,17 +18,17 @@ assignees: '' ## Possible Solution - - + ## Steps to Reproduce (for bugs) 1. -2. -3. -4. ## Context diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 4a7b218..d6d1162 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -7,14 +7,14 @@ assignees: '' --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +## Is your feature request related to a problem? Please describe. + -**Describe the solution you'd like** -A clear and concise description of what you want to happen. +## Describe the solution you'd like + -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +## Describe alternatives you've considered + -**Additional context** -Add any other context or screenshots about the feature request here. +## Additional context + From 1f661493164265ad3115bfdfda4907cd0189a5fc Mon Sep 17 00:00:00 2001 From: Egor Funtikov Date: Thu, 9 Mar 2023 11:49:52 +0300 Subject: [PATCH 366/548] [#17] Add pre-commit config --- .gitlint | 11 ++++++++++ .pre-commit-config.yaml | 45 +++++++++++++++++++++++++++++++++++++++++ Makefile | 10 ++++++++- 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 .gitlint create mode 100644 .pre-commit-config.yaml diff --git a/.gitlint b/.gitlint new file mode 100644 index 0000000..fda43fb --- /dev/null +++ b/.gitlint @@ -0,0 +1,11 @@ +[general] +fail-without-commits=True +regex-style-search=True +contrib=CC1 + +[title-match-regex] +regex=^\[\#[0-9Xx]+\]\s + +[ignore-by-title] +regex=^Release(.*) +ignore=title-match-regex \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1e89def --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,45 @@ +ci: + autofix_prs: false + +repos: + - repo: https://github.com/jorisroovers/gitlint + rev: v0.19.1 + hooks: + - id: gitlint + stages: [commit-msg] + - id: gitlint-ci + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: check-merge-conflict + - id: check-json + - id: check-xml + - id: check-yaml + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - id: end-of-file-fixer + exclude: ".key$" + + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.2 + hooks: + - id: shellcheck + + - repo: https://github.com/golangci/golangci-lint + rev: v1.51.2 + hooks: + - id: golangci-lint + + - repo: local + hooks: + - id: go-unit-tests + name: go unit tests + entry: make test + pass_filenames: false + types: [go] + language: system \ No newline at end of file diff --git a/Makefile b/Makefile index 41abb24..0fccc4e 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ BINDIR = bin DIRS = $(BINDIR) BINS = $(BINDIR)/frostfs-http-gw -.PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint version clean +.PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean # .deb package versioning OS_RELEASE = $(shell lsb_release -cs) @@ -109,6 +109,14 @@ docker/lint: --env HOME=/src \ golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint' +# Activate pre-commit hooks +pre-commit: + pre-commit install -t pre-commit -t commit-msg + +# Deactivate pre-commit hooks +unpre-commit: + pre-commit uninstall -t pre-commit -t commit-msg + # Print version version: @echo $(VERSION) From a8ec09e76a1770186b3fd47cf5b67189d1baf251 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 16 Mar 2023 11:40:08 +0300 Subject: [PATCH 367/548] [#22] Update system attributes prefix Signed-off-by: Denis Kirillov --- README.md | 22 ++-- docs/api.md | 108 ++++++++-------- downloader/download.go | 38 +----- downloader/download_test.go | 23 ---- downloader/head.go | 7 +- go.mod | 4 +- go.sum | 8 +- integration_test.go | 7 +- uploader/filter.go | 91 +------------ uploader/filter_test.go | 177 ++----------------------- uploader/upload.go | 62 ++------- utils/attributes.go | 250 +++++++++++++++++++++++++++++++++++- utils/attributes_test.go | 187 +++++++++++++++++++++++++++ utils/util.go | 26 ++++ 14 files changed, 562 insertions(+), 448 deletions(-) delete mode 100644 downloader/download_test.go create mode 100644 utils/attributes_test.go diff --git a/README.md b/README.md index de81301..3acf639 100644 --- a/README.md +++ b/README.md @@ -424,12 +424,12 @@ You can also add some attributes to your file using the following rules: "X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo: 100500" header to your request the resulting object will get "Ololo: 100500" attribute - * "X-Attribute-NEOFS-*" headers are special - (`-NEOFS-` part can also be `-neofs-` or`-Neofs-`), they're used to set internal - NeoFS attributes starting with `__NEOFS__` prefix, for these attributes all + * "X-Attribute-SYSTEM-*" headers are special + (`-SYSTEM-` part can also be `-system-` or`-System-` (and even legacy `-Neofs-` for some next releases)), they're used to set internal + FrostFS attributes starting with `__SYSTEM__` prefix, for these attributes all dashes get converted to underscores and all letters are capitalized. For - example, you can use "X-Attribute-NEOFS-Expiration-Epoch" header to set - `__NEOFS__EXPIRATION_EPOCH` attribute + example, you can use "X-Attribute-SYSTEM-Expiration-Epoch" header to set + `__SYSTEM__EXPIRATION_EPOCH` attribute * `FileName` attribute is set from multipart's `filename` if not set explicitly via `X-Attribute-FileName` header * `Timestamp` attribute can be set using gateway local time if using @@ -439,13 +439,13 @@ You can also add some attributes to your file using the following rules: --- **NOTE** -There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority): -1. `X-Attribute-Neofs-Expiration-Epoch: 100` -2. `X-Attribute-Neofs-Expiration-Duration: 24h30m` -3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797` -4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z` +There are some reserved headers type of `X-Attribute-SYSTEM-*` (headers are arranged in descending order of priority): +1. `X-Attribute-System-Expiration-Epoch: 100` +2. `X-Attribute-System-Expiration-Duration: 24h30m` +3. `X-Attribute-System-Expiration-Timestamp: 1637574797` +4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z` -which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way. +which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way. --- diff --git a/docs/api.md b/docs/api.md index 6a19465..78df766 100644 --- a/docs/api.md +++ b/docs/api.md @@ -56,21 +56,21 @@ Upload file as object with attributes to FrostFS. ###### Headers -| Header | Description | -|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------| -| Common headers | See [bearer token](#bearer-token). | -| `X-Attribute-Neofs-*` | Used to set system NeoFS object attributes
(e.g. use "X-Attribute-Neofs-Expiration-Epoch" to set `__NEOFS__EXPIRATION_EPOCH` attribute). | -| `X-Attribute-*` | Used to set regular object attributes
(e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | -| `Date` | This header is used to calculate the right `__NEOFS__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | +| Header | Description | +|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| +| Common headers | See [bearer token](#bearer-token). | +| `X-Attribute-System-*` | Used to set system FrostFS object attributes
(e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). | +| `X-Attribute-*` | Used to set regular object attributes
(e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | +| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | -There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority): +There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority): -1. `X-Attribute-Neofs-Expiration-Epoch: 100` -2. `X-Attribute-Neofs-Expiration-Duration: 24h30m` -3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797` -4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z` +1. `X-Attribute-System-Expiration-Epoch: 100` +2. `X-Attribute-System-Expiration-Duration: 24h30m` +3. `X-Attribute-System-Expiration-Timestamp: 1637574797` +4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z` -which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way. +which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way. If you don't specify the `X-Attribute-Timestamp` header the `Timestamp` attribute can be set anyway (see http-gw [configuration](gate-configuration.md#upload-header-section)). @@ -121,17 +121,17 @@ Get an object (payload and attributes) by an address. ###### Headers -| Header | Description | -|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------| -| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | -| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | -| `Content-Disposition` | Indicate how to browsers should treat file.
Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | -| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | -| `Content-Length` | Size of object payload. | -| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | -| `X-Owner-Id` | Base58 encoded owner ID. | -| `X-Container-Id` | Base58 encoded container ID. | -| `X-Object-Id` | Base58 encoded object ID. | +| Header | Description | +|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-System-*` | System FrostFS object attributes
(e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Disposition` | Indicate how to browsers should treat file.
Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | ###### Status codes @@ -157,16 +157,16 @@ Get an object attributes by an address. ###### Headers -| Header | Description | -|-----------------------|--------------------------------------------------------------------------------------------------------------------------| -| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | -| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | -| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | -| `Content-Length` | Size of object payload. | -| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | -| `X-Owner-Id` | Base58 encoded owner ID. | -| `X-Container-Id` | Base58 encoded container ID. | -| `X-Object-Id` | Base58 encoded object ID. | +| Header | Description | +|------------------------|------------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-System-*` | System FrostFS object attributes
(e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | ###### Status codes @@ -206,17 +206,17 @@ If more than one object is found, an arbitrary one will be returned. ###### Headers -| Header | Description | -|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------| -| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | -| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | -| `Content-Disposition` | Indicate how to browsers should treat file.
Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | -| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | -| `Content-Length` | Size of object payload. | -| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | -| `X-Owner-Id` | Base58 encoded owner ID. | -| `X-Container-Id` | Base58 encoded container ID. | -| `X-Object-Id` | Base58 encoded object ID. | +| Header | Description | +|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-System-*` | System FrostFS object attributes
(e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Disposition` | Indicate how to browsers should treat file.
Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | ###### Status codes @@ -243,16 +243,16 @@ If more than one object is found, an arbitrary one will be used to get attribute ###### Headers -| Header | Description | -|-----------------------|--------------------------------------------------------------------------------------------------------------------------| -| `X-Attribute-Neofs-*` | System NeoFS object attributes
(e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). | -| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | -| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | -| `Content-Length` | Size of object payload. | -| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | -| `X-Owner-Id` | Base58 encoded owner ID. | -| `X-Container-Id` | Base58 encoded container ID. | -| `X-Object-Id` | Base58 encoded object ID. | +| Header | Description | +|------------------------|------------------------------------------------------------------------------------------------------------------------------| +| `X-Attribute-System-*` | System FrostFS object attributes
(e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). | +| `X-Attribute-*` | Regular object attributes
(e.g. `My-Tag` set "X-Attribute-My-Tag" header). | +| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. | +| `Content-Length` | Size of object payload. | +| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). | +| `X-Owner-Id` | Base58 encoded owner ID. | +| `X-Container-Id` | Base58 encoded container ID. | +| `X-Object-Id` | Base58 encoded object ID. | ###### Status codes diff --git a/downloader/download.go b/downloader/download.go index d8f0bc2..6d24daa 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -14,8 +14,6 @@ import ( "strconv" "strings" "time" - "unicode" - "unicode/utf8" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" @@ -131,9 +129,9 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) { if !isValidToken(key) || !isValidValue(val) { continue } - if strings.HasPrefix(key, utils.SystemAttributePrefix) { - key = systemBackwardTranslator(key) - } + + key = utils.BackwardTransformIfSystem(key) + r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) switch key { case object.AttributeFileName: @@ -187,36 +185,6 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) { r.Response.SetBodyStream(rObj.Payload, int(payloadSize)) } -// systemBackwardTranslator is used to convert headers looking like '__NEOFS__ATTR_NAME' to 'Neofs-Attr-Name'. -func systemBackwardTranslator(key string) string { - // trim specified prefix '__NEOFS__' - key = strings.TrimPrefix(key, utils.SystemAttributePrefix) - - var res strings.Builder - res.WriteString("Neofs-") - - strs := strings.Split(key, "_") - for i, s := range strs { - s = title(strings.ToLower(s)) - res.WriteString(s) - if i != len(strs)-1 { - res.WriteString("-") - } - } - - return res.String() -} - -func title(str string) string { - if str == "" { - return "" - } - - r, size := utf8.DecodeRuneInString(str) - r0 := unicode.ToTitle(r) - return string(r0) + str[size:] -} - func bearerToken(ctx context.Context) *bearer.Token { if tkn, err := tokens.LoadBearerToken(ctx); err == nil { return tkn diff --git a/downloader/download_test.go b/downloader/download_test.go deleted file mode 100644 index a47d12a..0000000 --- a/downloader/download_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package downloader - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestSystemBackwardTranslator(t *testing.T) { - input := []string{ - "__NEOFS__EXPIRATION_EPOCH", - "__NEOFS__RANDOM_ATTR", - } - expected := []string{ - "Neofs-Expiration-Epoch", - "Neofs-Random-Attr", - } - - for i, str := range input { - res := systemBackwardTranslator(str) - require.Equal(t, expected[i], res) - } -} diff --git a/downloader/head.go b/downloader/head.go index 36f8b60..4615103 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -4,7 +4,6 @@ import ( "io" "net/http" "strconv" - "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" @@ -56,9 +55,9 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { if !isValidToken(key) || !isValidValue(val) { continue } - if strings.HasPrefix(key, utils.SystemAttributePrefix) { - key = systemBackwardTranslator(key) - } + + key = utils.BackwardTransformIfSystem(key) + r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) switch key { case object.AttributeTimestamp: diff --git a/go.mod b/go.mod index 4b7319e..0ce5791 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.18 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.0 github.com/prometheus/client_golang v1.13.0 diff --git a/go.sum b/go.sum index 585439e..7d4f24e 100644 --- a/go.sum +++ b/go.sum @@ -37,14 +37,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51 h1:l4+K1hN+NuWNtlZZoV8yRRP3Uu7PifL05ukEqKcb0Ks= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51/go.mod h1:n0DxKYulu2Ar73R6OcNF34LiL/Xa+iDR7GZuaOChbLE= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703 h1:lxe0DtZq/uFZVZu9apx6OcIXCJskQBMd/GVeYGKA3wA= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703/go.mod h1:gRd5iE5A84viily6AcNBsSlTx2XgoWrwRDz7z0MayDQ= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599 h1:mzGX2RX8R8H/tUqrUu1TcYk4QRDBcBIWGYscPncfLOQ= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599/go.mod h1:z7zcpGY+puI5puyy5oyFbf20vWp84WtslCxcr6/kv5c= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85 h1:TUcJ5A0C1gWi3bAhw4b+V+iVM3E9mbBOdJIWWkAPNxo= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85/go.mod h1:23fUGlEv/ImaOi3vck6vZj0v0b4hteOhLLPnVWHSQeA= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/integration_test.go b/integration_test.go index 0587273..40d908b 100644 --- a/integration_test.go +++ b/integration_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + containerv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -407,7 +408,11 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o if version >= versionWithNativeNames { var domain container.Domain domain.SetName(testContainerName) - container.WriteDomain(&cnr, domain) + + // currently node in aio image knows nothing about new sys attributes + // todo (@dkirillov): #2 use frostfs aio images that supports new attributes + cnr.SetAttribute(containerv2.SysAttributeNameNeoFS, domain.Name()) + cnr.SetAttribute(containerv2.SysAttributeZoneNeoFS, domain.Zone()) } var waitPrm pool.WaitParams diff --git a/uploader/filter.go b/uploader/filter.go index acab646..35de625 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -3,29 +3,12 @@ package uploader import ( "bytes" "fmt" - "math" - "strconv" - "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "github.com/valyala/fasthttp" "go.uber.org/zap" ) -var frostfsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")} - -func systemTranslator(key, prefix []byte) []byte { - // replace the specified prefix with `__NEOFS__` - key = bytes.Replace(key, prefix, []byte(utils.SystemAttributePrefix), 1) - - // replace `-` with `_` - key = bytes.ReplaceAll(key, []byte("-"), []byte("_")) - - // replace with uppercase - return bytes.ToUpper(key) -} - func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]string, error) { var err error result := make(map[string]string) @@ -45,13 +28,7 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st // removing attribute prefix clearKey := bytes.TrimPrefix(key, prefix) - // checks that it's a system NeoFS header - for _, system := range frostfsAttributeHeaderPrefixes { - if bytes.HasPrefix(clearKey, system) { - clearKey = systemTranslator(clearKey, system) - break - } - } + clearKey = utils.TransformIfSystem(clearKey) // checks that the attribute key is not empty if len(clearKey) == 0 { @@ -77,69 +54,3 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st return result, err } - -func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations, now time.Time) error { - expirationInEpoch := headers[object.SysAttributeExpEpoch] - - if timeRFC3339, ok := headers[utils.ExpirationRFC3339Attr]; ok { - expTime, err := time.Parse(time.RFC3339, timeRFC3339) - if err != nil { - return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, utils.ExpirationRFC3339Attr) - } - - if expTime.Before(now) { - return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, utils.ExpirationRFC3339Attr) - } - updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) - delete(headers, utils.ExpirationRFC3339Attr) - } - - if timestamp, ok := headers[utils.ExpirationTimestampAttr]; ok { - value, err := strconv.ParseInt(timestamp, 10, 64) - if err != nil { - return fmt.Errorf("couldn't parse value %s of header %s", timestamp, utils.ExpirationTimestampAttr) - } - expTime := time.Unix(value, 0) - - if expTime.Before(now) { - return fmt.Errorf("value %s of header %s must be in the future", timestamp, utils.ExpirationTimestampAttr) - } - updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) - delete(headers, utils.ExpirationTimestampAttr) - } - - if duration, ok := headers[utils.ExpirationDurationAttr]; ok { - expDuration, err := time.ParseDuration(duration) - if err != nil { - return fmt.Errorf("couldn't parse value %s of header %s", duration, utils.ExpirationDurationAttr) - } - if expDuration <= 0 { - return fmt.Errorf("value %s of header %s must be positive", expDuration, utils.ExpirationDurationAttr) - } - updateExpirationHeader(headers, epochDurations, expDuration) - delete(headers, utils.ExpirationDurationAttr) - } - - if expirationInEpoch != "" { - headers[object.SysAttributeExpEpoch] = expirationInEpoch - } - - return nil -} - -func updateExpirationHeader(headers map[string]string, durations *epochDurations, expDuration time.Duration) { - epochDuration := uint64(durations.msPerBlock) * durations.blockPerEpoch - currentEpoch := durations.currentEpoch - numEpoch := uint64(expDuration.Milliseconds()) / epochDuration - - if uint64(expDuration.Milliseconds())%epochDuration != 0 { - numEpoch++ - } - - expirationEpoch := uint64(math.MaxUint64) - if numEpoch < math.MaxUint64-currentEpoch { - expirationEpoch = currentEpoch + numEpoch - } - - headers[object.SysAttributeExpEpoch] = strconv.FormatUint(expirationEpoch, 10) -} diff --git a/uploader/filter_test.go b/uploader/filter_test.go index 9cb58c1..1190001 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -1,13 +1,8 @@ package uploader import ( - "math" - "strconv" "testing" - "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -28,8 +23,8 @@ func TestFilter(t *testing.T) { t.Run("duplicate system keys error", func(t *testing.T) { req := &fasthttp.RequestHeader{} req.DisableNormalizing() - req.Add("X-Attribute-Neofs-DupKey", "first-value") - req.Add("X-Attribute-Neofs-DupKey", "second-value") + req.Add("X-Attribute-System-DupKey", "first-value") + req.Add("X-Attribute-System-DupKey", "second-value") _, err := filterHeaders(log, req) require.Error(t, err) }) @@ -37,16 +32,16 @@ func TestFilter(t *testing.T) { req := &fasthttp.RequestHeader{} req.DisableNormalizing() - req.Set("X-Attribute-Neofs-Expiration-Epoch1", "101") - req.Set("X-Attribute-NEOFS-Expiration-Epoch2", "102") - req.Set("X-Attribute-neofs-Expiration-Epoch3", "103") + req.Set("X-Attribute-System-Expiration-Epoch1", "101") + req.Set("X-Attribute-SYSTEM-Expiration-Epoch2", "102") + req.Set("X-Attribute-system-Expiration-Epoch3", "103") req.Set("X-Attribute-MyAttribute", "value") expected := map[string]string{ - "__NEOFS__EXPIRATION_EPOCH1": "101", - "MyAttribute": "value", - "__NEOFS__EXPIRATION_EPOCH3": "103", - "__NEOFS__EXPIRATION_EPOCH2": "102", + "__SYSTEM__EXPIRATION_EPOCH1": "101", + "MyAttribute": "value", + "__SYSTEM__EXPIRATION_EPOCH3": "103", + "__SYSTEM__EXPIRATION_EPOCH2": "102", } result, err := filterHeaders(log, req) @@ -54,157 +49,3 @@ func TestFilter(t *testing.T) { require.Equal(t, expected, result) } - -func TestPrepareExpirationHeader(t *testing.T) { - tomorrow := time.Now().Add(24 * time.Hour) - tomorrowUnix := tomorrow.Unix() - tomorrowUnixNano := tomorrow.UnixNano() - tomorrowUnixMilli := tomorrowUnixNano / 1e6 - - epoch := "100" - duration := "24h" - timestampSec := strconv.FormatInt(tomorrowUnix, 10) - timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10) - timestampNano := strconv.FormatInt(tomorrowUnixNano, 10) - - defaultDurations := &epochDurations{ - currentEpoch: 10, - msPerBlock: 1000, - blockPerEpoch: 101, - } - - msPerBlock := defaultDurations.blockPerEpoch * uint64(defaultDurations.msPerBlock) - epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock - if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 { - epochPerDay++ - } - - defaultExpEpoch := strconv.FormatUint(defaultDurations.currentEpoch+epochPerDay, 10) - - for _, tc := range []struct { - name string - headers map[string]string - durations *epochDurations - err bool - expected map[string]string - }{ - { - name: "valid epoch", - headers: map[string]string{object.SysAttributeExpEpoch: epoch}, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid epoch, valid duration", - headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - utils.ExpirationDurationAttr: duration, - }, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid epoch, valid rfc3339", - headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339), - }, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid epoch, valid timestamp sec", - headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - utils.ExpirationTimestampAttr: timestampSec, - }, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid epoch, valid timestamp milli", - headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - utils.ExpirationTimestampAttr: timestampMilli, - }, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid epoch, valid timestamp nano", - headers: map[string]string{ - object.SysAttributeExpEpoch: epoch, - utils.ExpirationTimestampAttr: timestampNano, - }, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: epoch}, - }, - { - name: "valid timestamp sec", - headers: map[string]string{utils.ExpirationTimestampAttr: timestampSec}, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, - }, - { - name: "valid duration", - headers: map[string]string{utils.ExpirationDurationAttr: duration}, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, - }, - { - name: "valid rfc3339", - headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)}, - durations: defaultDurations, - expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch}, - }, - { - name: "valid max uint 64", - headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)}, - durations: &epochDurations{ - currentEpoch: math.MaxUint64 - 1, - msPerBlock: defaultDurations.msPerBlock, - blockPerEpoch: defaultDurations.blockPerEpoch, - }, - expected: map[string]string{object.SysAttributeExpEpoch: strconv.FormatUint(uint64(math.MaxUint64), 10)}, - }, - { - name: "invalid timestamp sec", - headers: map[string]string{utils.ExpirationTimestampAttr: "abc"}, - err: true, - }, - { - name: "invalid timestamp sec zero", - headers: map[string]string{utils.ExpirationTimestampAttr: "0"}, - err: true, - }, - { - name: "invalid duration", - headers: map[string]string{utils.ExpirationDurationAttr: "1d"}, - err: true, - }, - { - name: "invalid duration negative", - headers: map[string]string{utils.ExpirationDurationAttr: "-5h"}, - err: true, - }, - { - name: "invalid rfc3339", - headers: map[string]string{utils.ExpirationRFC3339Attr: "abc"}, - err: true, - }, - { - name: "invalid rfc3339 zero", - headers: map[string]string{utils.ExpirationRFC3339Attr: time.RFC3339}, - err: true, - }, - } { - t.Run(tc.name, func(t *testing.T) { - err := prepareExpirationHeader(tc.headers, tc.durations, time.Now()) - if tc.err { - require.Error(t, err) - } else { - require.NoError(t, err) - require.Equal(t, tc.expected, tc.headers) - } - }) - } -} diff --git a/uploader/upload.go b/uploader/upload.go index d455a13..4e21f08 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -3,7 +3,6 @@ package uploader import ( "context" "encoding/json" - "fmt" "io" "net/http" "strconv" @@ -38,12 +37,6 @@ type Uploader struct { containerResolver *resolver.ContainerResolver } -type epochDurations struct { - currentEpoch uint64 - msPerBlock int64 - blockPerEpoch uint64 -} - // Settings stores reloading parameters, so it has to provide atomic getters and setters. type Settings struct { defaultTimestamp atomic.Bool @@ -120,28 +113,20 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { response.Error(c, err.Error(), fasthttp.StatusBadRequest) return } - if needParseExpiration(filtered) { - epochDuration, err := getEpochDurations(c, u.pool) - if err != nil { - log.Error("could not get epoch durations from network info", zap.Error(err)) - response.Error(c, "could not get epoch durations from network info: "+err.Error(), fasthttp.StatusBadRequest) - return - } - now := time.Now() - if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { - if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { - log.Warn("could not parse client time", zap.String("Date header", string(rawHeader)), zap.Error(err)) - } else { - now = parsed - } + now := time.Now() + if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { + if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { + log.Warn("could not parse client time", zap.String("Date header", string(rawHeader)), zap.Error(err)) + } else { + now = parsed } + } - if err = prepareExpirationHeader(filtered, epochDuration, now); err != nil { - log.Error("could not parse expiration header", zap.Error(err)) - response.Error(c, "could not parse expiration header: "+err.Error(), fasthttp.StatusBadRequest) - return - } + if err = utils.PrepareExpirationHeader(c, u.pool, filtered, now); err != nil { + log.Error("could not prepare expiration header", zap.Error(err)) + response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) + return } attributes := make([]object.Attribute, 0, len(filtered)) @@ -246,28 +231,3 @@ func (pr *putResponse) encode(w io.Writer) error { enc.SetIndent("", "\t") return enc.Encode(pr) } - -func getEpochDurations(ctx context.Context, p *pool.Pool) (*epochDurations, error) { - networkInfo, err := p.NetworkInfo(ctx) - if err != nil { - return nil, err - } - - res := &epochDurations{ - currentEpoch: networkInfo.CurrentEpoch(), - msPerBlock: networkInfo.MsPerBlock(), - blockPerEpoch: networkInfo.EpochDuration(), - } - - if res.blockPerEpoch == 0 { - return nil, fmt.Errorf("EpochDuration is empty") - } - return res, nil -} - -func needParseExpiration(headers map[string]string) bool { - _, ok1 := headers[utils.ExpirationDurationAttr] - _, ok2 := headers[utils.ExpirationRFC3339Attr] - _, ok3 := headers[utils.ExpirationTimestampAttr] - return ok1 || ok2 || ok3 -} diff --git a/utils/attributes.go b/utils/attributes.go index 814d7b1..cfa3e3a 100644 --- a/utils/attributes.go +++ b/utils/attributes.go @@ -1,10 +1,250 @@ package utils +import ( + "bytes" + "context" + "errors" + "fmt" + "math" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" +) + const ( UserAttributeHeaderPrefix = "X-Attribute-" - SystemAttributePrefix = "__NEOFS__" - - ExpirationDurationAttr = SystemAttributePrefix + "EXPIRATION_DURATION" - ExpirationTimestampAttr = SystemAttributePrefix + "EXPIRATION_TIMESTAMP" - ExpirationRFC3339Attr = SystemAttributePrefix + "EXPIRATION_RFC3339" ) + +const ( + systemAttributePrefix = "__SYSTEM__" + + // deprecated: use systemAttributePrefix + systemAttributePrefixNeoFS = "__NEOFS__" +) + +type systemTransformer struct { + prefix string + backwardPrefix string + xAttrPrefixes [][]byte +} + +var transformers = []systemTransformer{ + { + prefix: systemAttributePrefix, + backwardPrefix: "System-", + xAttrPrefixes: [][]byte{[]byte("System-"), []byte("SYSTEM-"), []byte("system-")}, + }, + { + prefix: systemAttributePrefixNeoFS, + backwardPrefix: "Neofs-", + xAttrPrefixes: [][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")}, + }, +} + +func (t systemTransformer) existsExpirationAttributes(headers map[string]string) bool { + _, ok0 := headers[t.expirationEpochAttr()] + _, ok1 := headers[t.expirationDurationAttr()] + _, ok2 := headers[t.expirationTimestampAttr()] + _, ok3 := headers[t.expirationRFC3339Attr()] + return ok0 || ok1 || ok2 || ok3 +} + +func (t systemTransformer) expirationEpochAttr() string { + return t.prefix + "EXPIRATION_EPOCH" +} + +func (t systemTransformer) expirationDurationAttr() string { + return t.prefix + "EXPIRATION_DURATION" +} + +func (t systemTransformer) expirationTimestampAttr() string { + return t.prefix + "EXPIRATION_TIMESTAMP" +} + +func (t systemTransformer) expirationRFC3339Attr() string { + return t.prefix + "EXPIRATION_RFC3339" +} + +func (t systemTransformer) systemTranslator(key, prefix []byte) []byte { + // replace the specified prefix with system prefix + key = bytes.Replace(key, prefix, []byte(t.prefix), 1) + + // replace `-` with `_` + key = bytes.ReplaceAll(key, []byte("-"), []byte("_")) + + // replace with uppercase + return bytes.ToUpper(key) +} + +func (t systemTransformer) transformIfSystem(key []byte) ([]byte, bool) { + // checks that it's a system FrostFS header + for _, system := range t.xAttrPrefixes { + if bytes.HasPrefix(key, system) { + return t.systemTranslator(key, system), true + } + } + + return key, false +} + +// systemBackwardTranslator is used to convert headers looking like '__PREFIX__ATTR_NAME' to 'Prefix-Attr-Name'. +func (t systemTransformer) systemBackwardTranslator(key string) string { + // trim specified prefix '__PREFIX__' + key = strings.TrimPrefix(key, t.prefix) + + var res strings.Builder + res.WriteString(t.backwardPrefix) + + strs := strings.Split(key, "_") + for i, s := range strs { + s = title(strings.ToLower(s)) + res.WriteString(s) + if i != len(strs)-1 { + res.WriteString("-") + } + } + + return res.String() +} + +func (t systemTransformer) backwardTransformIfSystem(key string) (string, bool) { + if strings.HasPrefix(key, t.prefix) { + return t.systemBackwardTranslator(key), true + } + + return key, false +} + +func TransformIfSystem(key []byte) []byte { + for _, transformer := range transformers { + key, transformed := transformer.transformIfSystem(key) + if transformed { + return key + } + } + + return key +} + +func BackwardTransformIfSystem(key string) string { + for _, transformer := range transformers { + key, transformed := transformer.backwardTransformIfSystem(key) + if transformed { + return key + } + } + + return key +} + +func title(str string) string { + if str == "" { + return "" + } + + r, size := utf8.DecodeRuneInString(str) + r0 := unicode.ToTitle(r) + return string(r0) + str[size:] +} + +func PrepareExpirationHeader(ctx context.Context, p *pool.Pool, headers map[string]string, now time.Time) error { + formatsNum := 0 + index := -1 + for i, transformer := range transformers { + if transformer.existsExpirationAttributes(headers) { + formatsNum++ + index = i + } + } + + switch formatsNum { + case 0: + return nil + case 1: + epochDuration, err := GetEpochDurations(ctx, p) + if err != nil { + return fmt.Errorf("couldn't get epoch durations from network info: %w", err) + } + return transformers[index].prepareExpirationHeader(headers, epochDuration, now) + default: + return errors.New("both deprecated and new system attributes formats are used, please use only one") + } +} + +func (t systemTransformer) prepareExpirationHeader(headers map[string]string, epochDurations *EpochDurations, now time.Time) error { + expirationInEpoch := headers[t.expirationEpochAttr()] + + if timeRFC3339, ok := headers[t.expirationRFC3339Attr()]; ok { + expTime, err := time.Parse(time.RFC3339, timeRFC3339) + if err != nil { + return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, t.expirationRFC3339Attr()) + } + + if expTime.Before(now) { + return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, t.expirationRFC3339Attr()) + } + t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) + delete(headers, t.expirationRFC3339Attr()) + } + + if timestamp, ok := headers[t.expirationTimestampAttr()]; ok { + value, err := strconv.ParseInt(timestamp, 10, 64) + if err != nil { + return fmt.Errorf("couldn't parse value %s of header %s", timestamp, t.expirationTimestampAttr()) + } + expTime := time.Unix(value, 0) + + if expTime.Before(now) { + return fmt.Errorf("value %s of header %s must be in the future", timestamp, t.expirationTimestampAttr()) + } + t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now)) + delete(headers, t.expirationTimestampAttr()) + } + + if duration, ok := headers[t.expirationDurationAttr()]; ok { + expDuration, err := time.ParseDuration(duration) + if err != nil { + return fmt.Errorf("couldn't parse value %s of header %s", duration, t.expirationDurationAttr()) + } + if expDuration <= 0 { + return fmt.Errorf("value %s of header %s must be positive", expDuration, t.expirationDurationAttr()) + } + t.updateExpirationHeader(headers, epochDurations, expDuration) + delete(headers, t.expirationDurationAttr()) + } + + if expirationInEpoch != "" { + expEpoch, err := strconv.ParseUint(expirationInEpoch, 10, 64) + if err != nil { + return fmt.Errorf("parse expiration epoch '%s': %w", expirationInEpoch, err) + } + if expEpoch < epochDurations.CurrentEpoch { + return fmt.Errorf("expiration epoch '%d' must be greater than current epoch '%d'", expEpoch, epochDurations.CurrentEpoch) + } + + headers[t.expirationEpochAttr()] = expirationInEpoch + } + + return nil +} + +func (t systemTransformer) updateExpirationHeader(headers map[string]string, durations *EpochDurations, expDuration time.Duration) { + epochDuration := uint64(durations.MsPerBlock) * durations.BlockPerEpoch + currentEpoch := durations.CurrentEpoch + numEpoch := uint64(expDuration.Milliseconds()) / epochDuration + + if uint64(expDuration.Milliseconds())%epochDuration != 0 { + numEpoch++ + } + + expirationEpoch := uint64(math.MaxUint64) + if numEpoch < math.MaxUint64-currentEpoch { + expirationEpoch = currentEpoch + numEpoch + } + + headers[t.expirationEpochAttr()] = strconv.FormatUint(expirationEpoch, 10) +} diff --git a/utils/attributes_test.go b/utils/attributes_test.go new file mode 100644 index 0000000..a903b60 --- /dev/null +++ b/utils/attributes_test.go @@ -0,0 +1,187 @@ +package utils + +import ( + "math" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestPrepareExpirationHeader(t *testing.T) { + tomorrow := time.Now().Add(24 * time.Hour) + tomorrowUnix := tomorrow.Unix() + tomorrowUnixNano := tomorrow.UnixNano() + tomorrowUnixMilli := tomorrowUnixNano / 1e6 + + epoch := "100" + duration := "24h" + timestampSec := strconv.FormatInt(tomorrowUnix, 10) + timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10) + timestampNano := strconv.FormatInt(tomorrowUnixNano, 10) + + defaultDurations := &EpochDurations{ + CurrentEpoch: 10, + MsPerBlock: 1000, + BlockPerEpoch: 101, + } + + msPerBlock := defaultDurations.BlockPerEpoch * uint64(defaultDurations.MsPerBlock) + epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock + if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 { + epochPerDay++ + } + + defaultExpEpoch := strconv.FormatUint(defaultDurations.CurrentEpoch+epochPerDay, 10) + + for _, transformer := range transformers { + for _, tc := range []struct { + name string + headers map[string]string + durations *EpochDurations + err bool + expected map[string]string + }{ + { + name: "valid epoch", + headers: map[string]string{transformer.expirationEpochAttr(): epoch}, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + durations: defaultDurations, + }, + { + name: "valid epoch, valid duration", + headers: map[string]string{ + transformer.expirationEpochAttr(): epoch, + transformer.expirationDurationAttr(): duration, + }, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + }, + { + name: "valid epoch, valid rfc3339", + headers: map[string]string{ + transformer.expirationEpochAttr(): epoch, + transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339), + }, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + }, + { + name: "valid epoch, valid timestamp sec", + headers: map[string]string{ + transformer.expirationEpochAttr(): epoch, + transformer.expirationTimestampAttr(): timestampSec, + }, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + }, + { + name: "valid epoch, valid timestamp milli", + headers: map[string]string{ + transformer.expirationEpochAttr(): epoch, + transformer.expirationTimestampAttr(): timestampMilli, + }, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + }, + { + name: "valid epoch, valid timestamp nano", + headers: map[string]string{ + transformer.expirationEpochAttr(): epoch, + transformer.expirationTimestampAttr(): timestampNano, + }, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): epoch}, + }, + { + name: "valid timestamp sec", + headers: map[string]string{transformer.expirationTimestampAttr(): timestampSec}, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch}, + }, + { + name: "valid duration", + headers: map[string]string{transformer.expirationDurationAttr(): duration}, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch}, + }, + { + name: "valid rfc3339", + headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)}, + durations: defaultDurations, + expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch}, + }, + { + name: "valid max uint 64", + headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)}, + durations: &EpochDurations{ + CurrentEpoch: math.MaxUint64 - 1, + MsPerBlock: defaultDurations.MsPerBlock, + BlockPerEpoch: defaultDurations.BlockPerEpoch, + }, + expected: map[string]string{transformer.expirationEpochAttr(): strconv.FormatUint(uint64(math.MaxUint64), 10)}, + }, + { + name: "invalid timestamp sec", + headers: map[string]string{transformer.expirationTimestampAttr(): "abc"}, + err: true, + }, + { + name: "invalid timestamp sec zero", + headers: map[string]string{transformer.expirationTimestampAttr(): "0"}, + err: true, + }, + { + name: "invalid duration", + headers: map[string]string{transformer.expirationDurationAttr(): "1d"}, + err: true, + }, + { + name: "invalid duration negative", + headers: map[string]string{transformer.expirationDurationAttr(): "-5h"}, + err: true, + }, + { + name: "invalid rfc3339", + headers: map[string]string{transformer.expirationRFC3339Attr(): "abc"}, + err: true, + }, + { + name: "invalid rfc3339 zero", + headers: map[string]string{transformer.expirationRFC3339Attr(): time.RFC3339}, + err: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := transformer.prepareExpirationHeader(tc.headers, tc.durations, time.Now()) + if tc.err { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.expected, tc.headers) + } + }) + } + } +} + +func TestSystemBackwardTranslator(t *testing.T) { + input := []string{ + "__SYSTEM__EXPIRATION_EPOCH", + "__SYSTEM__RANDOM_ATTR", + "__NEOFS__EXPIRATION_EPOCH", + "__NEOFS__RANDOM_ATTR", + } + expected := []string{ + "System-Expiration-Epoch", + "System-Random-Attr", + "Neofs-Expiration-Epoch", + "Neofs-Random-Attr", + } + + for i, str := range input { + res := BackwardTransformIfSystem(str) + require.Equal(t, expected[i], res) + } +} diff --git a/utils/util.go b/utils/util.go index a83064c..d5a476a 100644 --- a/utils/util.go +++ b/utils/util.go @@ -2,9 +2,11 @@ package utils import ( "context" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" ) // GetContainerID decode container id, if it's not a valid container id @@ -17,3 +19,27 @@ func GetContainerID(ctx context.Context, containerID string, resolver *resolver. } return cnrID, err } + +type EpochDurations struct { + CurrentEpoch uint64 + MsPerBlock int64 + BlockPerEpoch uint64 +} + +func GetEpochDurations(ctx context.Context, p *pool.Pool) (*EpochDurations, error) { + networkInfo, err := p.NetworkInfo(ctx) + if err != nil { + return nil, err + } + + res := &EpochDurations{ + CurrentEpoch: networkInfo.CurrentEpoch(), + MsPerBlock: networkInfo.MsPerBlock(), + BlockPerEpoch: networkInfo.EpochDuration(), + } + + if res.BlockPerEpoch == 0 { + return nil, fmt.Errorf("EpochDuration is empty") + } + return res, nil +} From 81f7168a16889edea87d5bdb8b5eafd3b006ef11 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 16 Mar 2023 11:44:49 +0300 Subject: [PATCH 368/548] [#22] Update CHANGELOG.md Signed-off-by: Denis Kirillov --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dec0627..e3c86db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,16 @@ This document outlines major changes between releases. - Update neo-go to v0.101.0 (#8) - Update viper to v1.15.0 (#8) - Errors have become more detailed (#18) +- Update system attribute names (#22) + +### Updating from v0.26.0 + +To set system attributes use updated headers +(you can use old ones for now, but their support will be dropped in the future releases): + +* `X-Attribute-Neofs-*` -> `X-Attribute-System-*` +* `X-Attribute-NEOFS-*` -> `X-Attribute-SYSTEM-*` +* `X-Attribute-neofs-*` -> `X-Attribute-system-*` ## [0.26.0] - 2022-12-28 From 6f35d7198d15f1f464dbfac393f7271d6662e913 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 24 Mar 2023 12:25:49 +0300 Subject: [PATCH 369/548] [#24] Use build tags to run integration tests Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + Makefile | 5 +++++ downloader/reader_test.go | 2 ++ integration_test.go | 2 ++ tokens/bearer-token_test.go | 2 ++ uploader/filter_test.go | 2 ++ uploader/multipart_test.go | 2 ++ utils/attributes_test.go | 2 ++ 8 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3c86db..1dcd456 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ This document outlines major changes between releases. - Update viper to v1.15.0 (#8) - Errors have become more detailed (#18) - Update system attribute names (#22) +- Separate integration tests with build tags (#24) ### Updating from v0.26.0 diff --git a/Makefile b/Makefile index 0fccc4e..a4bdb1e 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,11 @@ docker/%: test: @go test ./... -cover +# Run integration tests +.PHONY: integration-test +integration-test: + @go test ./... -cover --tags=integration + # Run tests with race detection and produce coverage output cover: @go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic diff --git a/downloader/reader_test.go b/downloader/reader_test.go index 8d58185..09c990a 100644 --- a/downloader/reader_test.go +++ b/downloader/reader_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package downloader import ( diff --git a/integration_test.go b/integration_test.go index 40d908b..7a170a3 100644 --- a/integration_test.go +++ b/integration_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index b03147c..d5706cc 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package tokens import ( diff --git a/uploader/filter_test.go b/uploader/filter_test.go index 1190001..9d32b84 100644 --- a/uploader/filter_test.go +++ b/uploader/filter_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package uploader import ( diff --git a/uploader/multipart_test.go b/uploader/multipart_test.go index 9763f88..aad2b66 100644 --- a/uploader/multipart_test.go +++ b/uploader/multipart_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package uploader import ( diff --git a/utils/attributes_test.go b/utils/attributes_test.go index a903b60..251113b 100644 --- a/utils/attributes_test.go +++ b/utils/attributes_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package utils import ( From 7c16ffa250b8aaadfb898789d61429adb17434a0 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 24 Mar 2023 12:35:22 +0300 Subject: [PATCH 370/548] [#26] Fix pre-commit issues Signed-off-by: Denis Kirillov --- .gitlint | 2 +- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 6 ++-- Makefile | 2 +- README.md | 64 ++++++++++++++++----------------- debian/control | 1 - debian/copyright | 8 ++--- debian/frostfs-http-gw.postinst | 4 +-- debian/frostfs-http-gw.postrm | 0 debian/frostfs-http-gw.preinst | 0 debian/frostfs-http-gw.prerm | 0 debian/rules | 6 ++-- docs/gate-configuration.md | 10 +++--- 13 files changed, 51 insertions(+), 54 deletions(-) mode change 100644 => 100755 Makefile mode change 100644 => 100755 debian/frostfs-http-gw.postinst mode change 100644 => 100755 debian/frostfs-http-gw.postrm mode change 100644 => 100755 debian/frostfs-http-gw.preinst mode change 100644 => 100755 debian/frostfs-http-gw.prerm diff --git a/.gitlint b/.gitlint index fda43fb..e7218ac 100644 --- a/.gitlint +++ b/.gitlint @@ -8,4 +8,4 @@ regex=^\[\#[0-9Xx]+\]\s [ignore-by-title] regex=^Release(.*) -ignore=title-match-regex \ No newline at end of file +ignore=title-match-regex diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1e89def..4fde2a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,4 +42,4 @@ repos: entry: make test pass_filenames: false types: [go] - language: system \ No newline at end of file + language: system diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dcd456..dee5545 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,7 +139,7 @@ If you configure application using `.yaml` file change: ### Changed - Updated docs (#133, #140) -- Increased default read/write timeouts (#154) +- Increased default read/write timeouts (#154) - Updated SDK (#137, #139) - Updated go version to 1.17 (#143) - Improved error messages (#144) @@ -166,11 +166,11 @@ If you configure application using `.yaml` file change: - System headers format (#111) ### Added -- Different formats to set object's expiration: in epoch, duration, timestamp, +- Different formats to set object's expiration: in epoch, duration, timestamp, RFC3339 (#108) - Support of nodes priority (#115) -### Changed +### Changed - Updated testcontainers dependency (#100) ## [0.17.0] - 2021-11-15 diff --git a/Makefile b/Makefile old mode 100644 new mode 100755 index a4bdb1e..c8a1c23 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ OS_RELEASE = $(shell lsb_release -cs) PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \ sed "s/-/~/")-${OS_RELEASE} -.PHONY: debpackage debclean +.PHONY: debpackage debclean # Make all binaries all: $(BINS) diff --git a/README.md b/README.md index 3acf639..504768f 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,8 @@ can be done either via `-p` parameter or via `HTTP_GW_PEERS__ADDRESS` and `HTTP_GW_PEERS__WEIGHT` environment variables (the gate supports multiple FrostFS nodes with weighted load balancing). -If you launch HTTP gateway in bundle with [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env), -you can get the IP address of the node in the output of `make hosts` command +If you launch HTTP gateway in bundle with [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env), +you can get the IP address of the node in the output of `make hosts` command (with s0*.frostfs.devenv name). These two commands are functionally equivalent, they run the gate with one @@ -86,12 +86,12 @@ $ HTTP_GW_PEERS_0_ADDRESS=192.168.130.71:8080 HTTP_GW_PEERS_0_WEIGHT=1 HTTP_GW_P HTTP_GW_PEERS_2_ADDRESS=192.168.130.73:8080 HTTP_GW_PEERS_2_WEIGHT=1 HTTP_GW_PEERS_2_PRIORITY=2 \ frostfs-http-gw ``` -This command will make gateway use 192.168.130.71 while it is healthy. Otherwise, it will make the gateway use +This command will make gateway use 192.168.130.71 while it is healthy. Otherwise, it will make the gateway use 192.168.130.72 for 90% of requests and 192.168.130.73 for remaining 10%. ### Keys -You can provide a wallet via `--wallet` or `-w` flag. You can also specify the account address using `--address` -(if no address provided default one will be used). If wallet is used, you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt the wallet. +You can provide a wallet via `--wallet` or `-w` flag. You can also specify the account address using `--address` +(if no address provided default one will be used). If wallet is used, you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt the wallet. If no wallet provided, the gateway autogenerates a key pair it will use for FrostFS requests. ``` $ frostfs-http-gw -p $FROSTFS_NODE -w $WALLET_PATH --address $ACCOUNT_ADDRESS @@ -162,7 +162,7 @@ All timing options accept values with suffixes, so "15s" is 15 seconds and "2m" is 2 minutes. ### Zip streaming -The gateway supports downloading files by common prefix (like dir) in zip format. You can enable compression +The gateway supports downloading files by common prefix (like dir) in zip format. You can enable compression using config or `HTTP_GW_ZIP_COMPRESSION=true` environment variable. ### Logging @@ -172,7 +172,7 @@ HTTP_GW_LOGGER_LEVEL=debug ``` ### Yaml file -Configuration file is optional and can be used instead of environment variables/other parameters. +Configuration file is optional and can be used instead of environment variables/other parameters. It can be specified with `--config` parameter: ``` $ frostfs-http-gw --config your-config.yaml @@ -207,7 +207,7 @@ supported. ### Preparation -Before uploading or downloading a file make sure you have a prepared container. +Before uploading or downloading a file make sure you have a prepared container. You can create it with instructions below. Also, in case of downloading, you need to have a file inside a container. @@ -226,13 +226,13 @@ resolve_order: - nns ``` -2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env) +2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env) you can check if your container (e.g. with `container-name` name) is registered in NNS: ```shell $ curl -s --data '{"id":1,"jsonrpc":"2.0","method":"getcontractstate","params":[1]}' \ http://morph-chain.frostfs.devenv:30333 | jq -r '.result.hash' - + 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 $ docker exec -it morph_chain neo-go \ @@ -241,7 +241,7 @@ $ docker exec -it morph_chain neo-go \ resolve string:container-name.container int:16 \ | jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \ | base64 -d && echo - + 7f3vvkw4iTiS5ZZbu5BQXEmJtETWbi3uUjLNaSs29xrL ``` @@ -257,7 +257,7 @@ You can create a container via [frostfs-cli](https://git.frostfs.info/TrueCloudL ``` $ frostfs-cli -r $FROSTFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL ``` -where `$WALLET` is a path to user wallet, +where `$WALLET` is a path to user wallet, `$ACL` -- hex encoded basic ACL value or keywords 'private, 'public-read', 'public-read-write' and `$POLICY` -- QL-encoded or JSON-encoded placement policy or path to file with it @@ -267,17 +267,17 @@ $ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy ``` If you have launched nodes via [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env), -you can get the key value from `wallets/wallet.json` or write the path to +you can get the key value from `wallets/wallet.json` or write the path to the file `wallets/wallet.key`. #### Prepare a file in a container To create a file via [frostfs-cli](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases), run a command below: ``` -$ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID +$ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID ``` -where -`$KEY` -- the key, please read the information [above](#create-a-container), +where +`$KEY` -- the key, please read the information [above](#create-a-container), `$CID` -- container ID. For example: @@ -290,13 +290,13 @@ $ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json object put --file cat.png #### Requests -The following requests support GET/HEAD methods. +The following requests support GET/HEAD methods. ##### By IDs Basic downloading involves container ID and object ID and is done via GET -requests to `/get/$CID/$OID` path, where `$CID` is a container ID or its name if NNS is enabled, -`$OID` is an object's (i.e. your file's) ID. +requests to `/get/$CID/$OID` path, where `$CID` is a container ID or its name if NNS is enabled, +`$OID` is an object's (i.e. your file's) ID. For example: @@ -317,12 +317,12 @@ can be used as well. The generic syntax for it looks like this: ```/get_by_attribute/$CID/$ATTRIBUTE_NAME/$ATTRIBUTE_VALUE``` -where -`$CID` is a container ID or its name if NNS is enabled, +where +`$CID` is a container ID or its name if NNS is enabled, `$ATTRIBUTE_NAME` is the name of the attribute we want to use, `$ATTRIBUTE_VALUE` is the value of this attribute that the target object should have. -**NB!** The attribute key and value should be url encoded, i.e., if you want to download an object with the attribute value +**NB!** The attribute key and value should be url encoded, i.e., if you want to download an object with the attribute value `a cat`, the value in the request must be `a+cat`. In the same way with the attribute key. If you don't escape such values everything can still work (for example you can use `d@ta` without encoding) but it's HIGHLY RECOMMENDED to encode all your attributes. @@ -346,7 +346,7 @@ Some other user-defined attributes: $ wget http://localhost:8082/get_by_attribute/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/Ololo/100500 ``` -Or when the attribute includes special symbols: +Or when the attribute includes special symbols: ``` $ wget http://localhost:8082/get_by_attribute/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/Olo%2Blo/100500 # means Olo+lo ``` @@ -365,7 +365,7 @@ You can download some dir (files with the same prefix) in zip (it will be compre $ wget http://localhost:8082/zip/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/common/prefix ``` -**Note:** the objects must have a valid `FilePath` attribute (it should not contain trailing `/`), +**Note:** the objects must have a valid `FilePath` attribute (it should not contain trailing `/`), otherwise they will not be in the zip archive. You can upload file with this attribute using `curl`: ``` @@ -393,7 +393,7 @@ set of reply headers generated using the following rules: ##### Caching strategy -HTTP Gateway doesn't control caching (doesn't anything with the `Cache-Control` header). Caching strategy strictly +HTTP Gateway doesn't control caching (doesn't anything with the `Cache-Control` header). Caching strategy strictly depends on application use case. So it should be carefully done by proxy server. ### Uploading @@ -424,7 +424,7 @@ You can also add some attributes to your file using the following rules: "X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo: 100500" header to your request the resulting object will get "Ololo: 100500" attribute - * "X-Attribute-SYSTEM-*" headers are special + * "X-Attribute-SYSTEM-*" headers are special (`-SYSTEM-` part can also be `-system-` or`-System-` (and even legacy `-Neofs-` for some next releases)), they're used to set internal FrostFS attributes starting with `__SYSTEM__` prefix, for these attributes all dashes get converted to underscores and all letters are capitalized. For @@ -445,7 +445,7 @@ There are some reserved headers type of `X-Attribute-SYSTEM-*` (headers are arra 3. `X-Attribute-System-Expiration-Timestamp: 1637574797` 4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z` -which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way. +which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way. --- @@ -484,7 +484,7 @@ the corresponding header to the upload request. Accessing the ACL protected data works the same way. ##### Example -In order to generate a bearer token, you need to know the container owner key and +In order to generate a bearer token, you need to know the container owner key and the address of the sender who will do the request to FrostFS (in our case, it's a gateway wallet address). Suppose we have: @@ -492,7 +492,7 @@ Suppose we have: * **NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3** (token owner address) * **BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K** (container id) -Firstly, we need to encode the container id and the sender address to base64 (now it's base58). +Firstly, we need to encode the container id and the sender address to base64 (now it's base58). So use **base58** and **base64** utils. 1. Encoding container id: @@ -540,7 +540,7 @@ $ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w ./wa ``` Encoding to base64 to use via the header: ``` -$ base64 -w 0 signed.json +$ base64 -w 0 signed.json # output: Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw== ``` @@ -599,8 +599,8 @@ File **eacl.json**: ### Metrics and Pprof -If enabled, Prometheus metrics are available at `localhost:8084` endpoint -and Pprof at `localhost:8083/debug/pprof` by default. Host and port can be configured. +If enabled, Prometheus metrics are available at `localhost:8084` endpoint +and Pprof at `localhost:8083/debug/pprof` by default. Host and port can be configured. See [configuration](./docs/gate-configuration.md). ## Credits diff --git a/debian/control b/debian/control index 612be94..7924bc6 100644 --- a/debian/control +++ b/debian/control @@ -12,4 +12,3 @@ Package: frostfs-http-gw Architecture: any Depends: ${misc:Depends} Description: FrostFS HTTP Gateway bridges FrostFS internal protocol and HTTP standard. - diff --git a/debian/copyright b/debian/copyright index c278a3a..9ab3cb9 100644 --- a/debian/copyright +++ b/debian/copyright @@ -14,12 +14,12 @@ License: GPL-3 This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 3. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License - along with this program or at /usr/share/common-licenses/GPL-3. - If not, see . + along with this program or at /usr/share/common-licenses/GPL-3. + If not, see . diff --git a/debian/frostfs-http-gw.postinst b/debian/frostfs-http-gw.postinst old mode 100644 new mode 100755 index 85b6c0f..360ceef --- a/debian/frostfs-http-gw.postinst +++ b/debian/frostfs-http-gw.postinst @@ -29,8 +29,8 @@ case "$1" in chmod -f 0640 /etc/frostfs/$USERNAME/config.yaml || true fi USERDIR=$(getent passwd "frostfs-$USERNAME" | cut -d: -f6) - if ! dpkg-statoverride --list frostfs-$USERDIR >/dev/null; then - chown -f frostfs-$USERNAME: $USERDIR + if ! dpkg-statoverride --list frostfs-"$USERDIR" >/dev/null; then + chown -f frostfs-$USERNAME: "$USERDIR" fi ;; diff --git a/debian/frostfs-http-gw.postrm b/debian/frostfs-http-gw.postrm old mode 100644 new mode 100755 diff --git a/debian/frostfs-http-gw.preinst b/debian/frostfs-http-gw.preinst old mode 100644 new mode 100755 diff --git a/debian/frostfs-http-gw.prerm b/debian/frostfs-http-gw.prerm old mode 100644 new mode 100755 diff --git a/debian/rules b/debian/rules index 477128a..0554034 100755 --- a/debian/rules +++ b/debian/rules @@ -8,9 +8,7 @@ SERVICE = frostfs-http-gw dh $@ override_dh_installsystemd: - dh_installsystemd --no-enable --no-start $(SERVICE).service + dh_installsystemd --no-enable --no-start $(SERVICE).service override_dh_installchangelogs: - dh_installchangelogs -k CHANGELOG.md - - + dh_installchangelogs -k CHANGELOG.md diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index d955d2a..8323bdc 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -62,9 +62,9 @@ resolve_order: - nns - dns -connect_timeout: 5s +connect_timeout: 5s stream_timeout: 10s -request_timeout: 5s +request_timeout: 5s rebalance_timer: 30s pool_error_threshold: 100 ``` @@ -83,8 +83,8 @@ pool_error_threshold: 100 ```yaml wallet: - path: /path/to/wallet.json - address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP + path: /path/to/wallet.json + address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP passphrase: pwd ``` @@ -201,7 +201,7 @@ upload_header: ```yaml zip: - compression: false + compression: false ``` | Parameter | Type | SIGHUP reload | Default value | Description | From 162738e771d9e4d5bd9d05728fa6780bf96c7c70 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 30 Mar 2023 15:58:45 +0300 Subject: [PATCH 371/548] [#27] Update SDK to fix handling request canceling Signed-off-by: Denis Kirillov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0ce5791..3438651 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230329125804-552219b8e130 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.0 github.com/prometheus/client_golang v1.13.0 diff --git a/go.sum b/go.sum index 7d4f24e..c2ab03d 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02f git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85 h1:TUcJ5A0C1gWi3bAhw4b+V+iVM3E9mbBOdJIWWkAPNxo= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85/go.mod h1:23fUGlEv/ImaOi3vck6vZj0v0b4hteOhLLPnVWHSQeA= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230329125804-552219b8e130 h1:V+3dGwEXwEvvSvseMKn8S6ZEMNhxBBYrcyx+F7VaptM= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230329125804-552219b8e130/go.mod h1:23fUGlEv/ImaOi3vck6vZj0v0b4hteOhLLPnVWHSQeA= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 959213520e2ddbab081c1cdffa0c98303535b46b Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 17 Apr 2023 16:28:27 +0300 Subject: [PATCH 372/548] [#32] Update health metric values Now values are: 0 - undefined 1 - starting 2 - ready 3 - shutting down Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + app.go | 16 ++++++---------- metrics/metrics.go | 12 +++++++++++- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dee5545..5ad285b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ This document outlines major changes between releases. - Errors have become more detailed (#18) - Update system attribute names (#22) - Separate integration tests with build tags (#24) +- Changed values for `frostfs_http_gw_state_health` metric (#32) ### Updating from v0.26.0 diff --git a/app.go b/app.go index 825da8e..0f49477 100644 --- a/app.go +++ b/app.go @@ -62,15 +62,10 @@ type ( gateMetrics struct { logger *zap.Logger - provider GateMetricsProvider + provider *metrics.GateMetrics mu sync.RWMutex enabled bool } - - GateMetricsProvider interface { - SetHealth(int32) - Unregister() - } ) // WithLogger returns Option to set a specific logger. @@ -214,9 +209,10 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) { func (a *app) initMetrics() { gateMetricsProvider := metrics.NewGateMetrics(a.pool) a.metrics = newGateMetrics(a.log, gateMetricsProvider, a.cfg.GetBool(cfgPrometheusEnabled)) + a.metrics.SetHealth(metrics.HealthStatusStarting) } -func newGateMetrics(logger *zap.Logger, provider GateMetricsProvider, enabled bool) *gateMetrics { +func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics { if !enabled { logger.Warn("metrics are disabled") } @@ -236,7 +232,7 @@ func (m *gateMetrics) SetEnabled(enabled bool) { m.mu.Unlock() } -func (m *gateMetrics) SetHealth(status int32) { +func (m *gateMetrics) SetHealth(status metrics.HealthStatus) { m.mu.RLock() if !m.enabled { m.mu.RUnlock() @@ -250,7 +246,7 @@ func (m *gateMetrics) SetHealth(status int32) { func (m *gateMetrics) Shutdown() { m.mu.Lock() if m.enabled { - m.provider.SetHealth(0) + m.provider.SetHealth(metrics.HealthStatusShuttingDown) m.enabled = false } m.provider.Unregister() @@ -335,7 +331,7 @@ func (a *app) Wait() { } func (a *app) setHealthStatus() { - a.metrics.SetHealth(1) + a.metrics.SetHealth(metrics.HealthStatusReady) } func (a *app) Serve(ctx context.Context) { diff --git a/metrics/metrics.go b/metrics/metrics.go index a184588..4797fd6 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -31,6 +31,16 @@ const ( methodCreateSession = "create_session" ) +// HealthStatus of the gate application. +type HealthStatus int32 + +const ( + HealthStatusUndefined HealthStatus = 0 + HealthStatusStarting HealthStatus = 1 + HealthStatusReady HealthStatus = 2 + HealthStatusShuttingDown HealthStatus = 3 +) + type GateMetrics struct { stateMetrics poolMetricsCollector @@ -87,7 +97,7 @@ func (m stateMetrics) unregister() { prometheus.Unregister(m.healthCheck) } -func (m stateMetrics) SetHealth(s int32) { +func (m stateMetrics) SetHealth(s HealthStatus) { m.healthCheck.Set(float64(s)) } From cc37c34396f737c89cc0b27c04faf6ae86c51462 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 7 Apr 2023 18:14:31 +0300 Subject: [PATCH 373/548] [#29] metrics: Support dump descriptions Signed-off-by: Denis Kirillov --- Makefile | 8 +++ app.go | 12 ++++ metrics/desc.go | 136 +++++++++++++++++++++++++++++++++++++++++++ metrics/desc_test.go | 26 +++++++++ metrics/metrics.go | 98 +++++++++---------------------- 5 files changed, 211 insertions(+), 69 deletions(-) create mode 100644 metrics/desc.go create mode 100644 metrics/desc_test.go diff --git a/Makefile b/Makefile index c8a1c23..eb74c3e 100755 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ BUILD ?= $(shell date -u --iso=seconds) HUB_IMAGE ?= truecloudlab/frostfs-http-gw HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" +METRICS_DUMP_OUT ?= ./metrics-dump.json + # List of binaries to build. For now just one. BINDIR = bin DIRS = $(BINDIR) @@ -143,4 +145,10 @@ debpackage: debclean: dh clean +# Dump metrics (use METRICS_DUMP_OUT variable to override default out file './metrics-dump.json') +.PHONY: dump-metrics +dump-metrics: + @go test ./metrics -run TestDescribeAll --tags=dump_metrics --out=$(abspath $(METRICS_DUMP_OUT)) + + include help.mk diff --git a/app.go b/app.go index 0f49477..8522ef4 100644 --- a/app.go +++ b/app.go @@ -243,6 +243,17 @@ func (m *gateMetrics) SetHealth(status metrics.HealthStatus) { m.provider.SetHealth(status) } +func (m *gateMetrics) SetVersion(ver string) { + m.mu.RLock() + if !m.enabled { + m.mu.RUnlock() + return + } + m.mu.RUnlock() + + m.provider.SetVersion(ver) +} + func (m *gateMetrics) Shutdown() { m.mu.Lock() if m.enabled { @@ -325,6 +336,7 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*ecds func (a *app) Wait() { a.log.Info("starting application", zap.String("app_name", "frostfs-http-gw"), zap.String("version", Version)) + a.metrics.SetVersion(Version) a.setHealthStatus() <-a.webDone // wait for web-server to be stopped diff --git a/metrics/desc.go b/metrics/desc.go new file mode 100644 index 0000000..d904f8a --- /dev/null +++ b/metrics/desc.go @@ -0,0 +1,136 @@ +package metrics + +import ( + "encoding/json" + + "github.com/prometheus/client_golang/prometheus" +) + +var appMetricsDesc = map[string]map[string]Description{ + poolSubsystem: { + overallErrorsMetric: Description{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: overallErrorsMetric, + Help: "Total number of errors in pool", + }, + overallNodeErrorsMetric: Description{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: overallNodeErrorsMetric, + Help: "Total number of errors for connection in pool", + VariableLabels: []string{"node"}, + }, + overallNodeRequestsMetric: Description{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: overallNodeRequestsMetric, + Help: "Total number of requests to specific node in pool", + VariableLabels: []string{"node"}, + }, + currentErrorMetric: Description{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: currentErrorMetric, + Help: "Number of errors on current connections that will be reset after the threshold", + VariableLabels: []string{"node"}, + }, + avgRequestDurationMetric: Description{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: avgRequestDurationMetric, + Help: "Average request duration (in milliseconds) for specific method on node in pool", + VariableLabels: []string{"node", "method"}, + }, + }, + stateSubsystem: { + healthMetric: Description{ + Namespace: namespace, + Subsystem: stateSubsystem, + Name: healthMetric, + Help: "Current HTTP gateway state", + }, + versionInfoMetric: Description{ + Namespace: namespace, + Subsystem: stateSubsystem, + Name: versionInfoMetric, + Help: "Version of current FrostFS HTTP Gate instance", + VariableLabels: []string{"version"}, + }, + }, +} + +type Description struct { + Namespace string + Subsystem string + Name string + Help string + ConstantLabels []KeyValue + VariableLabels []string +} + +type KeyValue struct { + Key string `json:"key"` + Value string `json:"value"` +} + +func (d *Description) MarshalJSON() ([]byte, error) { + return json.Marshal(&struct { + FQName string `json:"name"` + Help string `json:"help"` + ConstantLabels []KeyValue `json:"constant_labels"` + VariableLabels []string `json:"variable_labels"` + }{ + FQName: d.BuildFQName(), + Help: d.Help, + ConstantLabels: d.ConstantLabels, + VariableLabels: d.VariableLabels, + }) +} + +func (d *Description) BuildFQName() string { + return prometheus.BuildFQName(d.Namespace, d.Subsystem, d.Name) +} + +func (d *Description) ConstLabelsMap() map[string]string { + constsLabels := make(map[string]string, len(d.ConstantLabels)) + for _, kv := range d.ConstantLabels { + constsLabels[kv.Key] = kv.Value + } + return constsLabels +} + +// DescribeAll returns descriptions for metrics. +func DescribeAll() []Description { + var list []Description + for _, m := range appMetricsDesc { + for _, description := range m { + list = append(list, description) + } + } + + return list +} + +func newOpts(description Description) prometheus.Opts { + return prometheus.Opts{ + Namespace: description.Namespace, + Subsystem: description.Subsystem, + Name: description.Name, + Help: description.Help, + ConstLabels: description.ConstLabelsMap(), + } +} + +func newGauge(description Description) prometheus.Gauge { + return prometheus.NewGauge( + prometheus.GaugeOpts(newOpts(description)), + ) +} + +func newGaugeVec(description Description) *prometheus.GaugeVec { + return prometheus.NewGaugeVec( + prometheus.GaugeOpts(newOpts(description)), + description.VariableLabels, + ) +} diff --git a/metrics/desc_test.go b/metrics/desc_test.go new file mode 100644 index 0000000..3c2d06b --- /dev/null +++ b/metrics/desc_test.go @@ -0,0 +1,26 @@ +//go:build dump_metrics + +package metrics + +import ( + "encoding/json" + "flag" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +var metricsPath = flag.String("out", "", "File to export http gateway metrics to.") + +func TestDescribeAll(t *testing.T) { + flag.Parse() + + require.NotEmpty(t, metricsPath, "flag 'out' must be provided to dump metrics description") + + data, err := json.Marshal(DescribeAll()) + require.NoError(t, err) + + err = os.WriteFile(*metricsPath, data, 0644) + require.NoError(t, err) +} diff --git a/metrics/metrics.go b/metrics/metrics.go index 4797fd6..e2b33da 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -13,7 +13,22 @@ const ( namespace = "frostfs_http_gw" stateSubsystem = "state" poolSubsystem = "pool" +) +const ( + healthMetric = "health" + versionInfoMetric = "version_info" +) + +const ( + overallErrorsMetric = "overall_errors" + overallNodeErrorsMetric = "overall_node_errors" + overallNodeRequestsMetric = "overall_node_requests" + currentErrorMetric = "current_errors" + avgRequestDurationMetric = "avg_request_duration" +) + +const ( methodGetBalance = "get_balance" methodPutContainer = "put_container" methodGetContainer = "get_container" @@ -48,6 +63,7 @@ type GateMetrics struct { type stateMetrics struct { healthCheck prometheus.Gauge + versionInfo *prometheus.GaugeVec } type poolMetricsCollector struct { @@ -80,93 +96,37 @@ func (g *GateMetrics) Unregister() { func newStateMetrics() *stateMetrics { return &stateMetrics{ - healthCheck: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: stateSubsystem, - Name: "health", - Help: "Current HTTP gateway state", - }), + healthCheck: newGauge(appMetricsDesc[stateSubsystem][healthMetric]), + versionInfo: newGaugeVec(appMetricsDesc[stateSubsystem][versionInfoMetric]), } } func (m stateMetrics) register() { prometheus.MustRegister(m.healthCheck) + prometheus.MustRegister(m.versionInfo) } func (m stateMetrics) unregister() { prometheus.Unregister(m.healthCheck) + prometheus.Unregister(m.versionInfo) } func (m stateMetrics) SetHealth(s HealthStatus) { m.healthCheck.Set(float64(s)) } +func (m stateMetrics) SetVersion(ver string) { + m.versionInfo.WithLabelValues(ver).Set(1) +} + func newPoolMetricsCollector(p *pool.Pool) *poolMetricsCollector { - overallErrors := prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: poolSubsystem, - Name: "overall_errors", - Help: "Total number of errors in pool", - }, - ) - - overallNodeErrors := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: poolSubsystem, - Name: "overall_node_errors", - Help: "Total number of errors for connection in pool", - }, - []string{ - "node", - }, - ) - - overallNodeRequests := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: poolSubsystem, - Name: "overall_node_requests", - Help: "Total number of requests to specific node in pool", - }, - []string{ - "node", - }, - ) - - currentErrors := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: poolSubsystem, - Name: "current_errors", - Help: "Number of errors on current connections that will be reset after the threshold", - }, - []string{ - "node", - }, - ) - - requestsDuration := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: poolSubsystem, - Name: "avg_request_duration", - Help: "Average request duration (in milliseconds) for specific method on node in pool", - }, - []string{ - "node", - "method", - }, - ) - return &poolMetricsCollector{ pool: p, - overallErrors: overallErrors, - overallNodeErrors: overallNodeErrors, - overallNodeRequests: overallNodeRequests, - currentErrors: currentErrors, - requestDuration: requestsDuration, + overallErrors: newGauge(appMetricsDesc[poolSubsystem][overallErrorsMetric]), + overallNodeErrors: newGaugeVec(appMetricsDesc[poolSubsystem][overallNodeErrorsMetric]), + overallNodeRequests: newGaugeVec(appMetricsDesc[poolSubsystem][overallNodeRequestsMetric]), + currentErrors: newGaugeVec(appMetricsDesc[poolSubsystem][currentErrorMetric]), + requestDuration: newGaugeVec(appMetricsDesc[poolSubsystem][avgRequestDurationMetric]), } } From 6c6fd0e9a52444b9f9d1f87eae094841d0650319 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 7 Apr 2023 18:16:53 +0300 Subject: [PATCH 374/548] [#29] Update CHANGELOG.md Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ad285b..2ec3ce4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This document outlines major changes between releases. ### Added - Multiple configs support (TrueCloudLab#12) +- Support dump metrics descriptions (#29) ### Changed - Update go version to 1.18 (TrueCloudLab#9) From 385f336a17a206e752b552c7d564297b27335b60 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 10 Apr 2023 12:22:11 +0300 Subject: [PATCH 375/548] [#29] Add type to metrics description Signed-off-by: Denis Kirillov --- go.mod | 2 +- metrics/desc.go | 21 +++++++++++++++++++-- metrics/desc_test.go | 13 ++++++++++++- metrics/metrics.go | 28 ++++++++++++++++------------ 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 3438651..80ecfce 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.0 github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_model v0.2.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 @@ -68,7 +69,6 @@ require ( github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/metrics/desc.go b/metrics/desc.go index d904f8a..65b47d2 100644 --- a/metrics/desc.go +++ b/metrics/desc.go @@ -4,17 +4,20 @@ import ( "encoding/json" "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" ) var appMetricsDesc = map[string]map[string]Description{ poolSubsystem: { overallErrorsMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: poolSubsystem, Name: overallErrorsMetric, Help: "Total number of errors in pool", }, overallNodeErrorsMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: poolSubsystem, Name: overallNodeErrorsMetric, @@ -22,6 +25,7 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"node"}, }, overallNodeRequestsMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: poolSubsystem, Name: overallNodeRequestsMetric, @@ -29,6 +33,7 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"node"}, }, currentErrorMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: poolSubsystem, Name: currentErrorMetric, @@ -36,6 +41,7 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"node"}, }, avgRequestDurationMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: poolSubsystem, Name: avgRequestDurationMetric, @@ -45,12 +51,14 @@ var appMetricsDesc = map[string]map[string]Description{ }, stateSubsystem: { healthMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: stateSubsystem, Name: healthMetric, Help: "Current HTTP gateway state", }, versionInfoMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: stateSubsystem, Name: versionInfoMetric, @@ -61,6 +69,7 @@ var appMetricsDesc = map[string]map[string]Description{ } type Description struct { + Type dto.MetricType Namespace string Subsystem string Name string @@ -76,11 +85,13 @@ type KeyValue struct { func (d *Description) MarshalJSON() ([]byte, error) { return json.Marshal(&struct { + Type string `json:"type"` FQName string `json:"name"` Help string `json:"help"` ConstantLabels []KeyValue `json:"constant_labels"` VariableLabels []string `json:"variable_labels"` }{ + Type: d.Type.String(), FQName: d.BuildFQName(), Help: d.Help, ConstantLabels: d.ConstantLabels, @@ -122,13 +133,19 @@ func newOpts(description Description) prometheus.Opts { } } -func newGauge(description Description) prometheus.Gauge { +func mustNewGauge(description Description) prometheus.Gauge { + if description.Type != dto.MetricType_GAUGE { + panic("invalid metric type") + } return prometheus.NewGauge( prometheus.GaugeOpts(newOpts(description)), ) } -func newGaugeVec(description Description) *prometheus.GaugeVec { +func mustNewGaugeVec(description Description) *prometheus.GaugeVec { + if description.Type != dto.MetricType_GAUGE { + panic("invalid metric type") + } return prometheus.NewGaugeVec( prometheus.GaugeOpts(newOpts(description)), description.VariableLabels, diff --git a/metrics/desc_test.go b/metrics/desc_test.go index 3c2d06b..b3e98ae 100644 --- a/metrics/desc_test.go +++ b/metrics/desc_test.go @@ -8,17 +8,28 @@ import ( "os" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/stretchr/testify/require" ) +type mock struct{} + +func (m mock) Statistic() pool.Statistic { + return pool.Statistic{} +} + var metricsPath = flag.String("out", "", "File to export http gateway metrics to.") func TestDescribeAll(t *testing.T) { + // to check correct metrics type mapping + _ = NewGateMetrics(mock{}) + flag.Parse() require.NotEmpty(t, metricsPath, "flag 'out' must be provided to dump metrics description") - data, err := json.Marshal(DescribeAll()) + desc := DescribeAll() + data, err := json.Marshal(desc) require.NoError(t, err) err = os.WriteFile(*metricsPath, data, 0644) diff --git a/metrics/metrics.go b/metrics/metrics.go index e2b33da..cf22099 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -56,6 +56,10 @@ const ( HealthStatusShuttingDown HealthStatus = 3 ) +type StatisticScraper interface { + Statistic() pool.Statistic +} + type GateMetrics struct { stateMetrics poolMetricsCollector @@ -67,7 +71,7 @@ type stateMetrics struct { } type poolMetricsCollector struct { - pool *pool.Pool + scraper StatisticScraper overallErrors prometheus.Gauge overallNodeErrors *prometheus.GaugeVec overallNodeRequests *prometheus.GaugeVec @@ -76,7 +80,7 @@ type poolMetricsCollector struct { } // NewGateMetrics creates new metrics for http gate. -func NewGateMetrics(p *pool.Pool) *GateMetrics { +func NewGateMetrics(p StatisticScraper) *GateMetrics { stateMetric := newStateMetrics() stateMetric.register() @@ -96,8 +100,8 @@ func (g *GateMetrics) Unregister() { func newStateMetrics() *stateMetrics { return &stateMetrics{ - healthCheck: newGauge(appMetricsDesc[stateSubsystem][healthMetric]), - versionInfo: newGaugeVec(appMetricsDesc[stateSubsystem][versionInfoMetric]), + healthCheck: mustNewGauge(appMetricsDesc[stateSubsystem][healthMetric]), + versionInfo: mustNewGaugeVec(appMetricsDesc[stateSubsystem][versionInfoMetric]), } } @@ -119,14 +123,14 @@ func (m stateMetrics) SetVersion(ver string) { m.versionInfo.WithLabelValues(ver).Set(1) } -func newPoolMetricsCollector(p *pool.Pool) *poolMetricsCollector { +func newPoolMetricsCollector(p StatisticScraper) *poolMetricsCollector { return &poolMetricsCollector{ - pool: p, - overallErrors: newGauge(appMetricsDesc[poolSubsystem][overallErrorsMetric]), - overallNodeErrors: newGaugeVec(appMetricsDesc[poolSubsystem][overallNodeErrorsMetric]), - overallNodeRequests: newGaugeVec(appMetricsDesc[poolSubsystem][overallNodeRequestsMetric]), - currentErrors: newGaugeVec(appMetricsDesc[poolSubsystem][currentErrorMetric]), - requestDuration: newGaugeVec(appMetricsDesc[poolSubsystem][avgRequestDurationMetric]), + scraper: p, + overallErrors: mustNewGauge(appMetricsDesc[poolSubsystem][overallErrorsMetric]), + overallNodeErrors: mustNewGaugeVec(appMetricsDesc[poolSubsystem][overallNodeErrorsMetric]), + overallNodeRequests: mustNewGaugeVec(appMetricsDesc[poolSubsystem][overallNodeRequestsMetric]), + currentErrors: mustNewGaugeVec(appMetricsDesc[poolSubsystem][currentErrorMetric]), + requestDuration: mustNewGaugeVec(appMetricsDesc[poolSubsystem][avgRequestDurationMetric]), } } @@ -152,7 +156,7 @@ func (m *poolMetricsCollector) register() { } func (m *poolMetricsCollector) updateStatistic() { - stat := m.pool.Statistic() + stat := m.scraper.Statistic() m.overallNodeErrors.Reset() m.overallNodeRequests.Reset() From 15b65b521ba99da1faa1eaa3ee243114a92bc261 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 19 Apr 2023 15:18:15 +0300 Subject: [PATCH 376/548] [#29] metrics: Use map for constant labels Signed-off-by: Denis Kirillov --- metrics/desc.go | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/metrics/desc.go b/metrics/desc.go index 65b47d2..f2ff4f4 100644 --- a/metrics/desc.go +++ b/metrics/desc.go @@ -74,22 +74,17 @@ type Description struct { Subsystem string Name string Help string - ConstantLabels []KeyValue + ConstantLabels prometheus.Labels VariableLabels []string } -type KeyValue struct { - Key string `json:"key"` - Value string `json:"value"` -} - func (d *Description) MarshalJSON() ([]byte, error) { return json.Marshal(&struct { - Type string `json:"type"` - FQName string `json:"name"` - Help string `json:"help"` - ConstantLabels []KeyValue `json:"constant_labels"` - VariableLabels []string `json:"variable_labels"` + Type string `json:"type"` + FQName string `json:"name"` + Help string `json:"help"` + ConstantLabels prometheus.Labels `json:"constant_labels,omitempty"` + VariableLabels []string `json:"variable_labels,omitempty"` }{ Type: d.Type.String(), FQName: d.BuildFQName(), @@ -103,14 +98,6 @@ func (d *Description) BuildFQName() string { return prometheus.BuildFQName(d.Namespace, d.Subsystem, d.Name) } -func (d *Description) ConstLabelsMap() map[string]string { - constsLabels := make(map[string]string, len(d.ConstantLabels)) - for _, kv := range d.ConstantLabels { - constsLabels[kv.Key] = kv.Value - } - return constsLabels -} - // DescribeAll returns descriptions for metrics. func DescribeAll() []Description { var list []Description @@ -129,7 +116,7 @@ func newOpts(description Description) prometheus.Opts { Subsystem: description.Subsystem, Name: description.Name, Help: description.Help, - ConstLabels: description.ConstLabelsMap(), + ConstLabels: description.ConstantLabels, } } From 9eeaf44163d8b167e0efa53af182a218412961c8 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 26 Apr 2023 09:51:37 +0300 Subject: [PATCH 377/548] [#39] Enabling gate metrics Signed-off-by: Denis Kirillov --- app.go | 1 + 1 file changed, 1 insertion(+) diff --git a/app.go b/app.go index 8522ef4..c2dabc0 100644 --- a/app.go +++ b/app.go @@ -219,6 +219,7 @@ func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled b return &gateMetrics{ logger: logger, provider: provider, + enabled: enabled, } } From ad05f1eb82faa15da9e8299ea8f337ded262d5e9 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 27 Apr 2023 17:04:01 +0300 Subject: [PATCH 378/548] [#40] Support impersonate bearer token Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + go.mod | 33 +++++++++++++++-------- go.sum | 76 +++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 78 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ec3ce4..00b5ac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This document outlines major changes between releases. ### Added - Multiple configs support (TrueCloudLab#12) - Support dump metrics descriptions (#29) +- Support impersonate bearer token (#40) ### Changed - Update go version to 1.18 (TrueCloudLab#9) diff --git a/go.mod b/go.mod index 80ecfce..53cb35e 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,15 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.18 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230329125804-552219b8e130 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230420104105-d0762d037d2c github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.0 github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_model v0.2.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 github.com/testcontainers/testcontainers-go v0.13.0 github.com/valyala/fasthttp v1.34.0 go.uber.org/atomic v1.10.0 @@ -30,8 +30,8 @@ require ( github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/containerd v1.6.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect @@ -42,12 +42,15 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -83,16 +86,24 @@ require ( github.com/urfave/cli v1.22.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/otel v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect + go.opentelemetry.io/otel/sdk v1.14.0 // indirect + go.opentelemetry.io/otel/trace v1.14.0 // indirect + go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.4.0 // indirect golang.org/x/exp v0.0.0-20221227203929-1b447090c38c // indirect - golang.org/x/net v0.4.0 // indirect + golang.org/x/net v0.7.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/term v0.3.0 // indirect - golang.org/x/text v0.5.0 // indirect - google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect - google.golang.org/grpc v1.52.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/grpc v1.53.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index c2ab03d..2a586e3 100644 --- a/go.sum +++ b/go.sum @@ -37,14 +37,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703 h1:lxe0DtZq/uFZVZu9apx6OcIXCJskQBMd/GVeYGKA3wA= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703/go.mod h1:gRd5iE5A84viily6AcNBsSlTx2XgoWrwRDz7z0MayDQ= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 h1:77lvdk0kMhnUgtnmqEcAPXPQaGlt24goMPu2+E5WRTk= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85/go.mod h1:sPyITTmQT662ZI38ud2aoE1SUCAr1mO5xV8P4nzLkKI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230329125804-552219b8e130 h1:V+3dGwEXwEvvSvseMKn8S6ZEMNhxBBYrcyx+F7VaptM= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230329125804-552219b8e130/go.mod h1:23fUGlEv/ImaOi3vck6vZj0v0b4hteOhLLPnVWHSQeA= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230420104105-d0762d037d2c h1:dSicW6Gbb7sZTb+eyHj6IXihCyoCp5lGz/Z4YqnW8Ak= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230420104105-d0762d037d2c/go.mod h1:TaJJOF3Uhuq8aqv2CrfuY2yhxePUinW35Xd3wfXLV/I= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= @@ -156,14 +156,15 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0Bsq github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= +github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= @@ -180,8 +181,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= @@ -387,6 +391,11 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -421,6 +430,8 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -521,6 +532,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= @@ -865,8 +878,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -936,7 +950,23 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= +go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 h1:/fXHZHGvro6MVqV34fJzDhi7sHGpX3Ej/Qjmfn003ho= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 h1:TKf2uAs2ueguzLaxOCBXNpHxfO/aC7PAdDsSH0IbeRQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 h1:sEL90JjOO/4yhquXl5zTAkLLsZ5+MycAgX99SDsxGc8= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0/go.mod h1:oCslUcizYdpKYyS9e8srZEqM6BB8fq41VJBjLAE6z1w= +go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= +go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= +go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= +go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -944,8 +974,8 @@ go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= @@ -1069,8 +1099,8 @@ golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1081,6 +1111,7 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1195,23 +1226,24 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1357,8 +1389,9 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= -google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1383,8 +1416,9 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= -google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From a945cdd42cbdbf4d29d6ab95a181dd038fd4d650 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 15 Mar 2023 11:07:44 +0300 Subject: [PATCH 379/548] [#20] get/head: Add tracing support Signed-off-by: Dmitrii Stepanov --- app.go | 41 ++++++++++++++++++- config/config.env | 4 ++ config/config.yaml | 4 ++ docs/gate-configuration.md | 18 +++++++++ downloader/download.go | 79 ++++++++++++++++++++++-------------- downloader/head.go | 47 ++++++++++++++-------- go.mod | 4 +- settings.go | 5 +++ utils/tracing.go | 82 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 234 insertions(+), 50 deletions(-) create mode 100644 utils/tracing.go diff --git a/app.go b/app.go index c2dabc0..487fdae 100644 --- a/app.go +++ b/app.go @@ -10,7 +10,9 @@ import ( "strconv" "sync" "syscall" + "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/downloader" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" @@ -166,6 +168,7 @@ func newApp(ctx context.Context, opt ...Option) App { a.initAppSettings() a.initResolver() a.initMetrics() + a.initTracing(ctx) return a } @@ -375,7 +378,7 @@ LOOP: case <-ctx.Done(): break LOOP case <-sigs: - a.configReload() + a.configReload(ctx) } } @@ -383,11 +386,22 @@ LOOP: a.metrics.Shutdown() a.stopServices() + a.shutdownTracing() close(a.webDone) } -func (a *app) configReload() { +func (a *app) shutdownTracing() { + const tracingShutdownTimeout = 5 * time.Second + shdnCtx, cancel := context.WithTimeout(context.Background(), tracingShutdownTimeout) + defer cancel() + + if err := tracing.Shutdown(shdnCtx); err != nil { + a.log.Warn("failed to shutdown tracing", zap.Error(err)) + } +} + +func (a *app) configReload(ctx context.Context) { a.log.Info("SIGHUP config reload started") if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) { a.log.Warn("failed to reload config because it's missed") @@ -418,6 +432,7 @@ func (a *app) configReload() { a.updateSettings() a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled)) + a.initTracing(ctx) a.setHealthStatus() a.log.Info("SIGHUP config reload completed") @@ -549,3 +564,25 @@ func (a *app) serverIndex(address string) int { } return -1 } + +func (a *app) initTracing(ctx context.Context) { + instanceID := "" + if len(a.servers) > 0 { + instanceID = a.servers[0].Address() + } + cfg := tracing.Config{ + Enabled: a.cfg.GetBool(cfgTracingEnabled), + Exporter: tracing.Exporter(a.cfg.GetString(cfgTracingExporter)), + Endpoint: a.cfg.GetString(cfgTracingEndpoint), + Service: "frostfs-http-gw", + InstanceID: instanceID, + Version: Version, + } + updated, err := tracing.Setup(ctx, cfg) + if err != nil { + a.log.Warn("failed to initialize tracing", zap.Error(err)) + } + if updated { + a.log.Info("tracing config updated") + } +} diff --git a/config/config.env b/config/config.env index 4f13bb0..2d5ea94 100644 --- a/config/config.env +++ b/config/config.env @@ -92,3 +92,7 @@ HTTP_GW_POOL_ERROR_THRESHOLD=100 # Enable zip compression to download files by common prefix. HTTP_GW_ZIP_COMPRESSION=false + +HTTP_GW_TRACING_ENABLED=true +HTTP_GW_TRACING_ENDPOINT="localhost:4317" +HTTP_GW_TRACING_EXPORTER="otlp_grpc" \ No newline at end of file diff --git a/config/config.yaml b/config/config.yaml index 0a117df..510cb43 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -9,6 +9,10 @@ pprof: prometheus: enabled: false # Enable metrics. address: localhost:8084 +tracing: + enabled: true + exporter: "otlp_grpc" + endpoint: "localhost:4317" logger: level: debug # Log level. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 8323bdc..0d0504f 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -52,6 +52,7 @@ $ cat http.log | `zip` | [ZIP configuration](#zip-section) | | `pprof` | [Pprof configuration](#pprof-section) | | `prometheus` | [Prometheus configuration](#prometheus-section) | +| `tracing` | [Tracing configuration](#tracing-section) | # General section @@ -238,3 +239,20 @@ prometheus: |-----------|----------|---------------|------------------|-----------------------------------------| | `enabled` | `bool` | yes | `false` | Flag to enable the service. | | `address` | `string` | yes | `localhost:8084` | Address that service listener binds to. | + +# `tracing` section + +Contains configuration for the `tracing` service. + +```yaml +tracing: + enabled: true + exporter: "otlp_grpc" + endpoint: "localhost:4317" +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|------------|----------|---------------|------------------|---------------------------------------------------------------| +| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | +| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | +| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | diff --git a/downloader/download.go b/downloader/download.go index 6d24daa..7d29a26 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -27,14 +27,15 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/atomic" "go.uber.org/zap" ) type request struct { *fasthttp.RequestCtx - appCtx context.Context - log *zap.Logger + log *zap.Logger } func isValidToken(s string) bool { @@ -88,40 +89,40 @@ func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (str return http.DetectContentType(buf), buf, err // to not lose io.EOF } -func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) { +func receiveFile(ctx context.Context, req request, clnt *pool.Pool, objectAddress oid.Address) { var ( err error dis = "inline" start = time.Now() filename string ) - if err = tokens.StoreBearerToken(r.RequestCtx); err != nil { - r.log.Error("could not fetch and store bearer token", zap.Error(err)) - response.Error(r.RequestCtx, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) + if err = tokens.StoreBearerToken(req.RequestCtx); err != nil { + req.log.Error("could not fetch and store bearer token", zap.Error(err)) + response.Error(req.RequestCtx, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } var prm pool.PrmObjectGet prm.SetAddress(objectAddress) - if btoken := bearerToken(r.RequestCtx); btoken != nil { + if btoken := bearerToken(req.RequestCtx); btoken != nil { prm.UseBearer(*btoken) } - rObj, err := clnt.GetObject(r.appCtx, prm) + rObj, err := clnt.GetObject(ctx, prm) if err != nil { - r.handleFrostFSErr(err, start) + req.handleFrostFSErr(err, start) return } // we can't close reader in this function, so how to do it? - if r.Request.URI().QueryArgs().GetBool("download") { + if req.Request.URI().QueryArgs().GetBool("download") { dis = "attachment" } payloadSize := rObj.Header.PayloadSize() - r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) + req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) var contentType string for _, attr := range rObj.Header.Attributes() { key := attr.Key() @@ -132,27 +133,27 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) { key = utils.BackwardTransformIfSystem(key) - r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) + req.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) switch key { case object.AttributeFileName: filename = val case object.AttributeTimestamp: value, err := strconv.ParseInt(val, 10, 64) if err != nil { - r.log.Info("couldn't parse creation date", + req.log.Info("couldn't parse creation date", zap.String("key", key), zap.String("val", val), zap.Error(err)) continue } - r.Response.Header.Set(fasthttp.HeaderLastModified, + req.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat)) case object.AttributeContentType: contentType = val } } - idsToResponse(&r.Response, &rObj.Header) + idsToResponse(&req.Response, &rObj.Header) if len(contentType) == 0 { // determine the Content-Type from the payload head @@ -162,8 +163,8 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) { return rObj.Payload, nil }) if err != nil && err != io.EOF { - r.log.Error("could not detect Content-Type from payload", zap.Error(err)) - response.Error(r.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) + req.log.Error("could not detect Content-Type from payload", zap.Error(err)) + response.Error(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -178,11 +179,11 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) { // if it implements io.Closer and that's useful for us. rObj.Payload = readCloser{headReader, rObj.Payload} } - r.SetContentType(contentType) + req.SetContentType(contentType) - r.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) + req.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) - r.Response.SetBodyStream(rObj.Payload, int(payloadSize)) + req.Response.SetBodyStream(rObj.Payload, int(payloadSize)) } func bearerToken(ctx context.Context) *bearer.Token { @@ -240,26 +241,35 @@ func New(ctx context.Context, params *utils.AppParams, settings *Settings) *Down func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { return &request{ RequestCtx: ctx, - appCtx: d.appCtx, log: log, } } // DownloadByAddress handles download requests using simple cid/oid format. func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { - d.byAddress(c, request.receiveFile) + d.byAddress(c, receiveFile) } // byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Pool, oid.Address)) { +func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, *pool.Pool, oid.Address)) { var ( idCnr, _ = c.UserValue("cid").(string) idObj, _ = c.UserValue("oid").(string) log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj)) ) - cnrID, err := utils.GetContainerID(d.appCtx, idCnr, d.containerResolver) + ctx, span := utils.StartHTTPServerSpan(d.appCtx, c, "GET Object", + trace.WithAttributes( + attribute.String("cid", idCnr), + attribute.String("oid", idObj), + )) + defer func() { + utils.SetHTTPTraceInfo(ctx, span, c) + span.End() + }() + + cnrID, err := utils.GetContainerID(ctx, idCnr, d.containerResolver) if err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) @@ -277,16 +287,16 @@ func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Poo addr.SetContainer(*cnrID) addr.SetObject(*objID) - f(*d.newRequest(c, log), d.pool, addr) + f(ctx, *d.newRequest(c, log), d.pool, addr) } // DownloadByAttribute handles attribute-based download requests. func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { - d.byAttribute(c, request.receiveFile) + d.byAttribute(c, receiveFile) } // byAttribute is a wrapper similar to byAddress. -func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.Pool, oid.Address)) { +func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, *pool.Pool, oid.Address)) { var ( scid, _ = c.UserValue("cid").(string) key, _ = url.QueryUnescape(c.UserValue("attr_key").(string)) @@ -294,7 +304,18 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.P log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ) - containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver) + ctx, span := utils.StartHTTPServerSpan(d.appCtx, c, "GET Object", + trace.WithAttributes( + attribute.String("attr_key", key), + attribute.String("attr_val", val), + attribute.String("cid", scid), + )) + defer func() { + utils.SetHTTPTraceInfo(ctx, span, c) + span.End() + }() + + containerID, err := utils.GetContainerID(ctx, scid, d.containerResolver) if err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) @@ -329,7 +350,7 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.P addrObj.SetContainer(*containerID) addrObj.SetObject(buf[0]) - f(*d.newRequest(c, log), d.pool, addrObj) + f(ctx, *d.newRequest(c, log), d.pool, addrObj) } func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) (pool.ResObjectSearch, error) { diff --git a/downloader/head.go b/downloader/head.go index 4615103..9745019 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -1,6 +1,7 @@ package downloader import ( + "context" "io" "net/http" "strconv" @@ -13,6 +14,8 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -25,15 +28,25 @@ const ( hdrContainerID = "X-Container-Id" ) -func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { +func headObject(ctx context.Context, req request, clnt *pool.Pool, objectAddress oid.Address) { + ctx, span := utils.StartHTTPServerSpan(ctx, req.RequestCtx, "HEAD Object", + trace.WithAttributes( + attribute.String("cid", objectAddress.Container().EncodeToString()), + attribute.String("oid", objectAddress.Object().EncodeToString()), + )) + defer func() { + utils.SetHTTPTraceInfo(ctx, span, req.RequestCtx) + span.End() + }() + var start = time.Now() - if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { - r.log.Error("could not fetch and store bearer token", zap.Error(err)) - response.Error(r.RequestCtx, "could not fetch and store bearer token", fasthttp.StatusBadRequest) + if err := tokens.StoreBearerToken(req.RequestCtx); err != nil { + req.log.Error("could not fetch and store bearer token", zap.Error(err)) + response.Error(req.RequestCtx, "could not fetch and store bearer token", fasthttp.StatusBadRequest) return } - btoken := bearerToken(r.RequestCtx) + btoken := bearerToken(req.RequestCtx) var prm pool.PrmObjectHead prm.SetAddress(objectAddress) @@ -41,13 +54,13 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { prm.UseBearer(*btoken) } - obj, err := clnt.HeadObject(r.appCtx, prm) + obj, err := clnt.HeadObject(ctx, prm) if err != nil { - r.handleFrostFSErr(err, start) + req.handleFrostFSErr(err, start) return } - r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10)) + req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10)) var contentType string for _, attr := range obj.Attributes() { key := attr.Key() @@ -58,24 +71,24 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { key = utils.BackwardTransformIfSystem(key) - r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) + req.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) switch key { case object.AttributeTimestamp: value, err := strconv.ParseInt(val, 10, 64) if err != nil { - r.log.Info("couldn't parse creation date", + req.log.Info("couldn't parse creation date", zap.String("key", key), zap.String("val", val), zap.Error(err)) continue } - r.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat)) + req.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat)) case object.AttributeContentType: contentType = val } } - idsToResponse(&r.Response, &obj) + idsToResponse(&req.Response, &obj) if len(contentType) == 0 { contentType, _, err = readContentType(obj.PayloadSize(), func(sz uint64) (io.Reader, error) { @@ -86,18 +99,18 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) { prmRange.UseBearer(*btoken) } - resObj, err := clnt.ObjectRange(r.appCtx, prmRange) + resObj, err := clnt.ObjectRange(ctx, prmRange) if err != nil { return nil, err } return &resObj, nil }) if err != nil && err != io.EOF { - r.handleFrostFSErr(err, start) + req.handleFrostFSErr(err, start) return } } - r.SetContentType(contentType) + req.SetContentType(contentType) } func idsToResponse(resp *fasthttp.Response, obj *object.Object) { @@ -110,10 +123,10 @@ func idsToResponse(resp *fasthttp.Response, obj *object.Object) { // HeadByAddress handles head requests using simple cid/oid format. func (d *Downloader) HeadByAddress(c *fasthttp.RequestCtx) { - d.byAddress(c, request.headObject) + d.byAddress(c, headObject) } // HeadByAttribute handles attribute-based head requests. func (d *Downloader) HeadByAttribute(c *fasthttp.RequestCtx) { - d.byAttribute(c, request.headObject) + d.byAttribute(c, headObject) } diff --git a/go.mod b/go.mod index 53cb35e..906e032 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,8 @@ require ( github.com/stretchr/testify v1.8.2 github.com/testcontainers/testcontainers-go v0.13.0 github.com/valyala/fasthttp v1.34.0 + go.opentelemetry.io/otel v1.14.0 + go.opentelemetry.io/otel/trace v1.14.0 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 ) @@ -86,13 +88,11 @@ require ( github.com/urfave/cli v1.22.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect go.opentelemetry.io/otel/sdk v1.14.0 // indirect - go.opentelemetry.io/otel/trace v1.14.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.4.0 // indirect diff --git a/settings.go b/settings.go index 8d569c0..19188fd 100644 --- a/settings.go +++ b/settings.go @@ -47,6 +47,11 @@ const ( cfgPprofEnabled = "pprof.enabled" cfgPprofAddress = "pprof.address" + // Tracing ... + cfgTracingEnabled = "tracing.enabled" + cfgTracingExporter = "tracing.exporter" + cfgTracingEndpoint = "tracing.endpoint" + // Pool config. cfgConTimeout = "connect_timeout" cfgStreamTimeout = "stream_timeout" diff --git a/utils/tracing.go b/utils/tracing.go new file mode 100644 index 0000000..4bb74df --- /dev/null +++ b/utils/tracing.go @@ -0,0 +1,82 @@ +package utils + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "github.com/valyala/fasthttp" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + "go.opentelemetry.io/otel/trace" +) + +type httpCarrier struct { + r *fasthttp.RequestCtx +} + +func (c *httpCarrier) Get(key string) string { + bytes := c.r.Request.Header.Peek(key) + if len(bytes) == 0 { + return "" + } + return string(bytes) +} + +func (c *httpCarrier) Set(key string, value string) { + c.r.Response.Header.Set(key, value) +} + +func (c *httpCarrier) Keys() []string { + dict := make(map[string]interface{}) + c.r.Request.Header.VisitAll( + func(key, value []byte) { + dict[string(key)] = true + }, + ) + c.r.Response.Header.VisitAll( + func(key, value []byte) { + dict[string(key)] = true + }, + ) + result := make([]string, 0, len(dict)) + for key := range dict { + result = append(result, key) + } + return result +} + +func extractHTTPTraceInfo(ctx context.Context, req *fasthttp.RequestCtx) context.Context { + if req == nil { + return ctx + } + carrier := &httpCarrier{r: req} + return tracing.Propagator.Extract(ctx, carrier) +} + +// SetHTTPTraceInfo saves trace headers to response. +func SetHTTPTraceInfo(ctx context.Context, span trace.Span, req *fasthttp.RequestCtx) { + if req == nil { + return + } + if err := req.Err(); err != nil { + span.SetStatus(codes.Error, err.Error()) + } + span.SetAttributes( + semconv.HTTPStatusCode(req.Response.StatusCode()), + ) + carrier := &httpCarrier{r: req} + tracing.Propagator.Inject(ctx, carrier) +} + +// StartHTTPServerSpan starts root HTTP server span. +func StartHTTPServerSpan(ctx context.Context, req *fasthttp.RequestCtx, operationName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + ctx = extractHTTPTraceInfo(ctx, req) + opts = append(opts, trace.WithAttributes( + attribute.String("http.client_address", req.RemoteAddr().String()), + attribute.String("http.path", string(req.Path())), + semconv.HTTPMethod(string(req.Method())), + semconv.RPCService("frostfs-http-gw"), + ), trace.WithSpanKind(trace.SpanKindServer)) + return tracing.StartSpanFromContext(ctx, operationName, opts...) +} From 37dbb29535edc667d0198510fc59fe47e9b6998c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 5 May 2023 18:13:04 +0300 Subject: [PATCH 380/548] [#45] Update SDK to fix impersonated token Signed-off-by: Denis Kirillov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 906e032..fe0be0d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230420104105-d0762d037d2c + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.0 github.com/prometheus/client_golang v1.13.0 diff --git a/go.sum b/go.sum index 2a586e3..60ecb9f 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02f git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230420104105-d0762d037d2c h1:dSicW6Gbb7sZTb+eyHj6IXihCyoCp5lGz/Z4YqnW8Ak= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230420104105-d0762d037d2c/go.mod h1:TaJJOF3Uhuq8aqv2CrfuY2yhxePUinW35Xd3wfXLV/I= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd h1:HxacVl1Lc2RrfxAE13AGkp1tR/Mf4DDP6TgrgbLP5fQ= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd/go.mod h1:TaJJOF3Uhuq8aqv2CrfuY2yhxePUinW35Xd3wfXLV/I= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 8c3c3782f5b253e4588a0a79c15c8fda702c533a Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Fri, 5 May 2023 11:19:35 +0300 Subject: [PATCH 381/548] [#30] add object name resolving Signed-off-by: Pavel Pogodaev --- .gitignore | 1 + Makefile | 9 +- api/layer/tree_service.go | 22 +++ api/tree.go | 17 ++ app.go | 41 +++-- config/config.env | 3 + config/config.yaml | 4 + downloader/download.go | 59 ++++++- downloader/head.go | 14 +- go.mod | 19 ++- go.sum | 26 +-- internal/frostfs/services/tree_client_grpc.go | 114 +++++++++++++ .../services/tree_client_grpc_signature.go | 29 ++++ settings.go | 3 + syncTree.sh | 21 +++ tokens/bearer-token.go | 11 ++ tree/tree.go | 156 ++++++++++++++++++ 17 files changed, 507 insertions(+), 42 deletions(-) create mode 100644 api/layer/tree_service.go create mode 100644 api/tree.go create mode 100644 internal/frostfs/services/tree_client_grpc.go create mode 100644 internal/frostfs/services/tree_client_grpc_signature.go create mode 100755 syncTree.sh create mode 100644 tree/tree.go diff --git a/.gitignore b/.gitignore index c4a98d8..c1ca465 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ bin temp /plugins/ /vendor/ +internal/frostfs/services/tree .test.env *~ diff --git a/Makefile b/Makefile index eb74c3e..aeea3a8 100755 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ METRICS_DUMP_OUT ?= ./metrics-dump.json BINDIR = bin DIRS = $(BINDIR) BINS = $(BINDIR)/frostfs-http-gw +SYNCDIR = internal/frostfs/services/tree .PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean @@ -27,8 +28,7 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ # Make all binaries all: $(BINS) - -$(BINS): $(DIRS) dep +$(BINS): sync-tree $(DIRS) dep @echo "⇒ Build $@" CGO_ENABLED=0 \ go build -v -trimpath \ @@ -39,6 +39,10 @@ $(DIRS): @echo "⇒ Ensure dir: $@" @mkdir -p $@ +# Synchronize tree service +sync-tree: + @./syncTree.sh + # Pull go dependencies dep: @printf "⇒ Download requirements: " @@ -132,6 +136,7 @@ version: clean: rm -rf vendor rm -rf $(BINDIR) + rm -rf $(SYNCDIR) # Package for Debian debpackage: diff --git a/api/layer/tree_service.go b/api/layer/tree_service.go new file mode 100644 index 0000000..9852257 --- /dev/null +++ b/api/layer/tree_service.go @@ -0,0 +1,22 @@ +package layer + +import ( + "context" + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/api" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" +) + +// TreeService provide interface to interact with tree service using s3 data models. +type TreeService interface { + GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) +} + +var ( + // ErrNodeNotFound is returned from Tree service in case of not found error. + ErrNodeNotFound = errors.New("not found") + + // ErrNodeAccessDenied is returned from Tree service in case of access denied error. + ErrNodeAccessDenied = errors.New("access denied") +) diff --git a/api/tree.go b/api/tree.go new file mode 100644 index 0000000..4d16cc7 --- /dev/null +++ b/api/tree.go @@ -0,0 +1,17 @@ +package api + +import ( + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +// NodeVersion represent node from tree service. +type NodeVersion struct { + BaseNodeVersion + DeleteMarker bool +} + +// BaseNodeVersion is minimal node info from tree service. +// Basically used for "system" object. +type BaseNodeVersion struct { + OID oid.ID +} diff --git a/app.go b/app.go index 487fdae..a989c90 100644 --- a/app.go +++ b/app.go @@ -2,7 +2,6 @@ package main import ( "context" - "crypto/ecdsa" "fmt" "net/http" "os" @@ -14,9 +13,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/downloader" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" @@ -30,6 +31,8 @@ import ( "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) type ( @@ -37,6 +40,7 @@ type ( log *zap.Logger logLevel zap.AtomicLevel pool *pool.Pool + key *keys.PrivateKey owner *user.ID cfg *viper.Viper webServer *fasthttp.Server @@ -93,7 +97,6 @@ func WithConfig(c *viper.Viper) Option { func newApp(ctx context.Context, opt ...Option) App { var ( - key *ecdsa.PrivateKey err error ) @@ -120,17 +123,17 @@ func newApp(ctx context.Context, opt ...Option) App { a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - key, err = getFrostFSKey(a) + a.key, err = getFrostFSKey(a) if err != nil { a.log.Fatal("failed to get frostfs credentials", zap.Error(err)) } var owner user.ID - user.IDFromKey(&owner, key.PublicKey) + user.IDFromKey(&owner, a.key.PrivateKey.PublicKey) a.owner = &owner var prm pool.InitParameters - prm.SetKey(key) + prm.SetKey(&a.key.PrivateKey) prm.SetNodeDialTimeout(a.cfg.GetDuration(cfgConTimeout)) prm.SetNodeStreamTimeout(a.cfg.GetDuration(cfgStreamTimeout)) prm.SetHealthcheckTimeout(a.cfg.GetDuration(cfgReqTimeout)) @@ -277,7 +280,7 @@ func remove(list []string, element string) []string { return list } -func getFrostFSKey(a *app) (*ecdsa.PrivateKey, error) { +func getFrostFSKey(a *app) (*keys.PrivateKey, error) { walletPath := a.cfg.GetString(cfgWalletPath) if len(walletPath) == 0 { @@ -286,7 +289,7 @@ func getFrostFSKey(a *app) (*ecdsa.PrivateKey, error) { if err != nil { return nil, err } - return &key.PrivateKey, nil + return key, nil } w, err := wallet.NewWalletFromFile(walletPath) if err != nil { @@ -304,7 +307,7 @@ func getFrostFSKey(a *app) (*ecdsa.PrivateKey, error) { return getKeyFromWallet(w, address, password) } -func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*ecdsa.PrivateKey, error) { +func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*keys.PrivateKey, error) { var addr util.Uint160 var err error @@ -334,7 +337,7 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*ecds return nil, fmt.Errorf("couldn't decrypt account: %w", err) } - return &acc.PrivateKey().PrivateKey, nil + return acc.PrivateKey(), nil } func (a *app) Wait() { @@ -351,8 +354,9 @@ func (a *app) setHealthStatus() { } func (a *app) Serve(ctx context.Context) { + treeClient := a.initTree(ctx) uploadRoutes := uploader.New(ctx, a.AppParams(), a.settings.Uploader) - downloadRoutes := downloader.New(ctx, a.AppParams(), a.settings.Downloader) + downloadRoutes := downloader.New(ctx, a.AppParams(), a.settings.Downloader, treeClient) // Configure router. a.configureRouter(uploadRoutes, downloadRoutes) @@ -475,8 +479,8 @@ func (a *app) configureRouter(uploadRoutes *uploader.Uploader, downloadRoutes *d } r.POST("/upload/{cid}", a.logger(uploadRoutes.Upload)) a.log.Info("added path /upload/{cid}") - r.GET("/get/{cid}/{oid}", a.logger(downloadRoutes.DownloadByAddress)) - r.HEAD("/get/{cid}/{oid}", a.logger(downloadRoutes.HeadByAddress)) + r.GET("/get/{cid}/{oid:*}", a.logger(downloadRoutes.DownloadByAddressOrBucketName)) + r.HEAD("/get/{cid}/{oid:*}", a.logger(downloadRoutes.HeadByAddressOrBucketName)) a.log.Info("added path /get/{cid}/{oid}") r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.DownloadByAttribute)) r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.HeadByAttribute)) @@ -565,6 +569,19 @@ func (a *app) serverIndex(address string) int { return -1 } +func (a *app) initTree(ctx context.Context) *tree.Tree { + treeServiceEndpoint := a.cfg.GetString(cfgTreeServiceEndpoint) + grpcDialOpt := grpc.WithTransportCredentials(insecure.NewCredentials()) + treeGRPCClient, err := services.NewTreeServiceClientGRPC(ctx, treeServiceEndpoint, a.key, grpcDialOpt) + if err != nil { + a.log.Fatal("failed to create tree service", zap.Error(err)) + } + treeService := tree.NewTree(treeGRPCClient) + a.log.Info("init tree service", zap.String("endpoint", treeServiceEndpoint)) + + return treeService +} + func (a *app) initTracing(ctx context.Context) { instanceID := "" if len(a.servers) > 0 { diff --git a/config/config.env b/config/config.env index 2d5ea94..4dc9bb2 100644 --- a/config/config.env +++ b/config/config.env @@ -93,6 +93,9 @@ HTTP_GW_POOL_ERROR_THRESHOLD=100 # Enable zip compression to download files by common prefix. HTTP_GW_ZIP_COMPRESSION=false +# Endpoint of the tree service. Must be provided. Can be one of the node address (from the `peers` section). +HTTP_GW_TREE_SERVICE=grpc://s01.frostfs.devenv:8080 + HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENDPOINT="localhost:4317" HTTP_GW_TRACING_EXPORTER="otlp_grpc" \ No newline at end of file diff --git a/config/config.yaml b/config/config.yaml index 510cb43..a71c69d 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -93,6 +93,10 @@ resolve_order: upload_header: use_default_timestamp: false # Create timestamp for object if it isn't provided by header. +# Endpoint of the tree service. Must be provided. Can be one of the node address (from the `peers` section). +tree: + service: 127.0.0.1:8080 + connect_timeout: 5s # Timeout to dial node. stream_timeout: 10s # Timeout for individual operations in streaming RPC. request_timeout: 5s # Timeout to check node health during rebalance. diff --git a/downloader/download.go b/downloader/download.go index 7d29a26..b7c242b 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -18,6 +18,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -212,6 +213,7 @@ type Downloader struct { pool *pool.Pool containerResolver *resolver.ContainerResolver settings *Settings + tree *tree.Tree } // Settings stores reloading parameters, so it has to provide atomic getters and setters. @@ -228,13 +230,14 @@ func (s *Settings) SetZipCompression(val bool) { } // New creates an instance of Downloader using specified options. -func New(ctx context.Context, params *utils.AppParams, settings *Settings) *Downloader { +func New(ctx context.Context, params *utils.AppParams, settings *Settings, tree *tree.Tree) *Downloader { return &Downloader{ appCtx: ctx, log: params.Logger, pool: params.Pool, settings: settings, containerResolver: params.Resolver, + tree: tree, } } @@ -245,9 +248,16 @@ func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *requ } } -// DownloadByAddress handles download requests using simple cid/oid format. -func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { - d.byAddress(c, receiveFile) +// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. +func (d *Downloader) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { + test, _ := c.UserValue("oid").(string) + var id oid.ID + err := id.DecodeString(test) + if err != nil { + d.byBucketname(c, receiveFile) + } else { + d.byAddress(c, receiveFile) + } } // byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that @@ -290,6 +300,47 @@ func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(context.Context, r f(ctx, *d.newRequest(c, log), d.pool, addr) } +// byBucketname is a wrapper for function (e.g. request.headObject, request.receiveFile) that +// prepares request and object address to it. +func (d *Downloader) byBucketname(c *fasthttp.RequestCtx, f func(context.Context, request, *pool.Pool, oid.Address)) { + var ( + bucketname = c.UserValue("cid").(string) + key = c.UserValue("oid").(string) + log = d.log.With(zap.String("bucketname", bucketname), zap.String("key", key)) + ) + + cnrID, err := utils.GetContainerID(d.appCtx, bucketname, d.containerResolver) + if err != nil { + log.Error("wrong container id", zap.Error(err)) + response.Error(c, "wrong container id", fasthttp.StatusBadRequest) + return + } + + ctx, err := tokens.StoreBearerTokenAppCtx(c, d.appCtx) + if err != nil { + log.Error("could not fetch and store bearer token", zap.Error(err)) + response.Error(c, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + foundOid, err := d.tree.GetLatestVersion(ctx, cnrID, key) + if err != nil { + log.Error("object wasn't found", zap.Error(err)) + response.Error(c, "object wasn't found", fasthttp.StatusNotFound) + return + } + if foundOid.DeleteMarker { + log.Error("object was deleted") + response.Error(c, "object deleted", fasthttp.StatusNotFound) + return + } + var addr oid.Address + addr.SetContainer(*cnrID) + addr.SetObject(foundOid.OID) + + f(ctx, *d.newRequest(c, log), d.pool, addr) +} + // DownloadByAttribute handles attribute-based download requests. func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { d.byAttribute(c, receiveFile) diff --git a/downloader/head.go b/downloader/head.go index 9745019..a81a275 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -121,9 +121,17 @@ func idsToResponse(resp *fasthttp.Response, obj *object.Object) { resp.Header.Set(hdrContainerID, cnrID.String()) } -// HeadByAddress handles head requests using simple cid/oid format. -func (d *Downloader) HeadByAddress(c *fasthttp.RequestCtx) { - d.byAddress(c, headObject) +// HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format. +func (d *Downloader) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { + test, _ := c.UserValue("oid").(string) + var id oid.ID + + err := id.DecodeString(test) + if err != nil { + d.byBucketname(c, headObject) + } else { + d.byAddress(c, headObject) + } } // HeadByAttribute handles attribute-based head requests. diff --git a/go.mod b/go.mod index fe0be0d..21737d4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 + git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.0 @@ -18,11 +19,12 @@ require ( go.opentelemetry.io/otel/trace v1.14.0 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 + google.golang.org/grpc v1.53.0 + google.golang.org/protobuf v1.28.1 ) require ( git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb // indirect - git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/hrw v1.2.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect @@ -56,7 +58,7 @@ require ( github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/klauspost/compress v1.15.0 // indirect + github.com/klauspost/compress v1.16.4 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -95,16 +97,15 @@ require ( go.opentelemetry.io/otel/sdk v1.14.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.4.0 // indirect + golang.org/x/crypto v0.8.0 // indirect golang.org/x/exp v0.0.0-20221227203929-1b447090c38c // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/term v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + golang.org/x/time v0.3.0 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 60ecb9f..90996f3 100644 --- a/go.sum +++ b/go.sum @@ -586,8 +586,9 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= +github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1005,8 +1006,8 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1099,8 +1100,8 @@ golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1226,13 +1227,13 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1242,15 +1243,16 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/frostfs/services/tree_client_grpc.go b/internal/frostfs/services/tree_client_grpc.go new file mode 100644 index 0000000..f25588a --- /dev/null +++ b/internal/frostfs/services/tree_client_grpc.go @@ -0,0 +1,114 @@ +package services + +import ( + "context" + "fmt" + "strings" + + grpcService "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services/tree" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "google.golang.org/grpc" +) + +type GetNodeByPathResponseInfoWrapper struct { + response *grpcService.GetNodeByPathResponse_Info +} + +func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { + res := make([]tree.Meta, len(n.response.Meta)) + for i, value := range n.response.Meta { + res[i] = value + } + return res +} + +type GetSubTreeResponseBodyWrapper struct { + response *grpcService.GetSubTreeResponse_Body +} + +func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta { + res := make([]tree.Meta, len(n.response.Meta)) + for i, value := range n.response.Meta { + res[i] = value + } + return res +} + +type ServiceClientGRPC struct { + key *keys.PrivateKey + conn *grpc.ClientConn + service grpcService.TreeServiceClient +} + +func NewTreeServiceClientGRPC(ctx context.Context, addr string, key *keys.PrivateKey, grpcOpts ...grpc.DialOption) (*ServiceClientGRPC, error) { + conn, err := grpc.Dial(addr, grpcOpts...) + if err != nil { + return nil, fmt.Errorf("did not connect: %v", err) + } + + c := grpcService.NewTreeServiceClient(conn) + if _, err = c.Healthcheck(ctx, &grpcService.HealthcheckRequest{}); err != nil { + return nil, fmt.Errorf("healthcheck: %w", err) + } + + return &ServiceClientGRPC{ + key: key, + conn: conn, + service: c, + }, nil +} + +func (c *ServiceClientGRPC) GetNodes(ctx context.Context, p *tree.GetNodesParams) ([]tree.NodeResponse, error) { + request := &grpcService.GetNodeByPathRequest{ + Body: &grpcService.GetNodeByPathRequest_Body{ + ContainerId: p.CnrID[:], + TreeId: p.TreeID, + Path: p.Path, + Attributes: p.Meta, + PathAttribute: tree.FileNameKey, + LatestOnly: p.LatestOnly, + AllAttributes: p.AllAttrs, + BearerToken: getBearer(ctx), + }, + } + + if err := c.signRequest(request.Body, func(key, sign []byte) { + request.Signature = &grpcService.Signature{ + Key: key, + Sign: sign, + } + }); err != nil { + return nil, err + } + + resp, err := c.service.GetNodeByPath(ctx, request) + if err != nil { + return nil, handleError("failed to get node by path", err) + } + + res := make([]tree.NodeResponse, len(resp.GetBody().GetNodes())) + for i, info := range resp.GetBody().GetNodes() { + res[i] = GetNodeByPathResponseInfoWrapper{info} + } + + return res, nil +} + +func getBearer(ctx context.Context) []byte { + token, err := tokens.LoadBearerToken(ctx) + if err != nil { + return nil + } + return token.Marshal() +} + +func handleError(msg string, err error) error { + if strings.Contains(err.Error(), "not found") { + return fmt.Errorf("%w: %s", tree.ErrNodeNotFound, err.Error()) + } else if strings.Contains(err.Error(), "is denied by") { + return fmt.Errorf("%w: %s", tree.ErrNodeAccessDenied, err.Error()) + } + return fmt.Errorf("%s: %w", msg, err) +} diff --git a/internal/frostfs/services/tree_client_grpc_signature.go b/internal/frostfs/services/tree_client_grpc_signature.go new file mode 100644 index 0000000..9dd38f9 --- /dev/null +++ b/internal/frostfs/services/tree_client_grpc_signature.go @@ -0,0 +1,29 @@ +/*REMOVE THIS AFTER SIGNATURE WILL BE AVAILABLE IN TREE CLIENT FROM FROSTFS NODE*/ +package services + +import ( + crypto "git.frostfs.info/TrueCloudLab/frostfs-crypto" + "google.golang.org/protobuf/proto" +) + +func (c *ServiceClientGRPC) signData(buf []byte, f func(key, sign []byte)) error { + // crypto package should not be used outside of API libraries (see neofs-node#491). + // For now tree service does not include into SDK Client nor SDK Pool, so there is no choice. + // When SDK library adopts Tree service client, this should be dropped. + sign, err := crypto.Sign(&c.key.PrivateKey, buf) + if err != nil { + return err + } + + f(c.key.PublicKey().Bytes(), sign) + return nil +} + +func (c *ServiceClientGRPC) signRequest(requestBody proto.Message, f func(key, sign []byte)) error { + buf, err := proto.Marshal(requestBody) + if err != nil { + return err + } + + return c.signData(buf, f) +} diff --git a/settings.go b/settings.go index 19188fd..87c56bc 100644 --- a/settings.go +++ b/settings.go @@ -59,6 +59,9 @@ const ( cfgRebalance = "rebalance_timer" cfgPoolErrorThreshold = "pool_error_threshold" + // Grpc path to tree service. + cfgTreeServiceEndpoint = "tree.service" + // Logger. cfgLoggerLevel = "logger.level" diff --git a/syncTree.sh b/syncTree.sh new file mode 100755 index 0000000..98d16a0 --- /dev/null +++ b/syncTree.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +mkdir -p internal/frostfs/services/tree 2>/dev/null + +REVISION="f07d4158f50ed5c7f44cc0bc224c3d03edf27f3b" + +echo "tree service revision ${REVISION}" + +# regexp below find all link to source code files which end with ".pb.go" and retrieve the file names +# we use `[^.]*` as non greedy workaround for `.*` +FILES=$(curl -s https://git.frostfs.info/TrueCloudLab/frostfs-node/src/commit/${REVISION}/pkg/services/tree | sed -n "s,.*\"/TrueCloudLab/frostfs-node/src/commit/${REVISION}/pkg/services/tree/\([^.]*\.pb\.go\)\".*,\1,p") + +for file in $FILES; do + if [[ $file == *"frostfs"* ]]; then + echo "skip '$file'" + continue + else + echo "sync '$file' in tree service" + fi + curl -s "https://git.frostfs.info/TrueCloudLab/frostfs-node/raw/commit/${REVISION}/pkg/services/tree/${file}" -o "./internal/frostfs/services/tree/${file}" +done \ No newline at end of file diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index 6cd98ca..f43cb85 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -60,6 +60,17 @@ func StoreBearerToken(ctx *fasthttp.RequestCtx) error { return nil } +// StoreBearerTokenAppCtx extracts a bearer token from the header or cookie and stores +// it in the application context. +func StoreBearerTokenAppCtx(ctx *fasthttp.RequestCtx, appCtx context.Context) (context.Context, error) { + tkn, err := fetchBearerToken(ctx) + if err != nil { + return nil, err + } + newCtx := context.WithValue(appCtx, bearerTokenKey, tkn) + return newCtx, nil +} + // LoadBearerToken returns a bearer token stored in the context given (if it's // present there). func LoadBearerToken(ctx context.Context) (*bearer.Token, error) { diff --git a/tree/tree.go b/tree/tree.go new file mode 100644 index 0000000..84b6707 --- /dev/null +++ b/tree/tree.go @@ -0,0 +1,156 @@ +package tree + +import ( + "context" + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/api/layer" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +type ( + Tree struct { + service ServiceClient + } + + // ServiceClient is a client to interact with tree service. + // Each method must return ErrNodeNotFound or ErrNodeAccessDenied if relevant. + ServiceClient interface { + GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error) + } + + treeNode struct { + ObjID oid.ID + Meta map[string]string + } + + GetNodesParams struct { + CnrID cid.ID + TreeID string + Path []string + Meta []string + LatestOnly bool + AllAttrs bool + } +) + +var ( + // ErrNodeNotFound is returned from ServiceClient in case of not found error. + ErrNodeNotFound = layer.ErrNodeNotFound + + // ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error. + ErrNodeAccessDenied = layer.ErrNodeAccessDenied +) + +const ( + FileNameKey = "FileName" +) + +const ( + oidKV = "OID" + + // keys for delete marker nodes. + isDeleteMarkerKV = "IsDeleteMarker" + + // versionTree -- ID of a tree with object versions. + versionTree = "version" + + separator = "/" +) + +// NewTree creates instance of Tree using provided address and create grpc connection. +func NewTree(service ServiceClient) *Tree { + return &Tree{service: service} +} + +type Meta interface { + GetKey() string + GetValue() []byte +} + +type NodeResponse interface { + GetMeta() []Meta +} + +func newTreeNode(nodeInfo NodeResponse) (*treeNode, error) { + treeNode := &treeNode{ + Meta: make(map[string]string, len(nodeInfo.GetMeta())), + } + + for _, kv := range nodeInfo.GetMeta() { + switch kv.GetKey() { + case oidKV: + if err := treeNode.ObjID.DecodeString(string(kv.GetValue())); err != nil { + return nil, err + } + default: + treeNode.Meta[kv.GetKey()] = string(kv.GetValue()) + } + } + + return treeNode, nil +} + +func (n *treeNode) Get(key string) (string, bool) { + value, ok := n.Meta[key] + return value, ok +} + +func (n *treeNode) FileName() (string, bool) { + value, ok := n.Meta[FileNameKey] + return value, ok +} + +func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { + treeNode, err := newTreeNode(node) + if err != nil { + return nil, fmt.Errorf("invalid tree node: %w", err) + } + + return newNodeVersionFromTreeNode(treeNode), nil +} + +func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { + _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) + + version := &api.NodeVersion{ + BaseNodeVersion: api.BaseNodeVersion{ + OID: treeNode.ObjID, + }, + DeleteMarker: isDeleteMarker, + } + + return version +} + +func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { + meta := []string{oidKV, isDeleteMarkerKV} + path := pathFromName(objectName) + + p := &GetNodesParams{ + CnrID: *cnrID, + TreeID: versionTree, + Path: path, + Meta: meta, + LatestOnly: true, + AllAttrs: false, + } + nodes, err := c.service.GetNodes(ctx, p) + if err != nil { + return nil, err + } + + if len(nodes) == 0 { + return nil, layer.ErrNodeNotFound + } + + return newNodeVersion(nodes[0]) +} + +// pathFromName splits name by '/'. +func pathFromName(objectName string) []string { + return strings.Split(objectName, separator) +} From f7784db14620e369df44327398b0a0ee1421a462 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 15 May 2023 14:29:52 +0300 Subject: [PATCH 382/548] [#40] Update forming bearer token instruction Signed-off-by: Denis Kirillov --- README.md | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 504768f..1e050af 100644 --- a/README.md +++ b/README.md @@ -484,43 +484,26 @@ the corresponding header to the upload request. Accessing the ACL protected data works the same way. ##### Example -In order to generate a bearer token, you need to know the container owner key and +In order to generate a bearer token, you need to have wallet (which will be used to sign the token) and the address of the sender who will do the request to FrostFS (in our case, it's a gateway wallet address). Suppose we have: -* **KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr** (container owner key) -* **NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3** (token owner address) -* **BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K** (container id) +* **NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3** (token owner (gateway address)) Firstly, we need to encode the container id and the sender address to base64 (now it's base58). So use **base58** and **base64** utils. -1. Encoding container id: -``` -$ echo 'BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K' | base58 --decode | base64 -# output: mRnZWzewzxjzIPa7Fqlfqdl3TM1KpJ0YnsXsEhafJJg= -``` - -2. Encoding token owner id: +1. Encoding token owner id: ``` $ echo 'NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3' | base58 --decode | base64 # output: NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg== ``` -Now, we can form a Bearer token (10000 is liftetime expiration in epoch) and save it to **bearer.json**: +2. Form a Bearer token (10000 is lifetime expiration in epoch) and save it to **bearer.json**: ``` { "body": { - "eaclTable": { - "version": { - "major": 0, - "minor": 0 - }, - "containerID": { - "value": "mRnZWzewzxjzIPa7Fqlfqdl3TM1KpJ0YnsXsEhafJJg=" - }, - "records": [] - }, + "allowImpersonate": true, "ownerID": { "value": "NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg==" }, @@ -534,11 +517,12 @@ Now, we can form a Bearer token (10000 is liftetime expiration in epoch) and sav } ``` -Next, sign it with the container owner key: +3. Sign it with the wallet: ``` $ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w ./wallet.json ``` -Encoding to base64 to use via the header: + +4. Encode to base64 to use in header: ``` $ base64 -w 0 signed.json # output: Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw== From 3844ac83e6f14deb53300f22191fea5cc635682d Mon Sep 17 00:00:00 2001 From: Artem Tataurov Date: Thu, 18 May 2023 17:26:40 +0300 Subject: [PATCH 383/548] [#35] Update prometheus to v1.15.0 Signed-off-by: Artem Tataurov --- CHANGELOG.md | 1 + go.mod | 15 +++++++-------- go.sum | 24 +++++++++++++++--------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00b5ac8..3995307 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This document outlines major changes between releases. - Support impersonate bearer token (#40) ### Changed +- Update prometheus to v1.15.0 (#35) - Update go version to 1.18 (TrueCloudLab#9) - Update neo-go to v0.101.0 (#8) - Update viper to v1.15.0 (#8) diff --git a/go.mod b/go.mod index 21737d4..20c669d 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.0 - github.com/prometheus/client_golang v1.13.0 - github.com/prometheus/client_model v0.2.0 + github.com/prometheus/client_golang v1.15.0 + github.com/prometheus/client_model v0.3.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.2 @@ -20,7 +20,7 @@ require ( go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 google.golang.org/grpc v1.53.0 - google.golang.org/protobuf v1.28.1 + google.golang.org/protobuf v1.30.0 ) require ( @@ -50,7 +50,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect @@ -60,7 +60,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/klauspost/compress v1.16.4 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/sys/mount v0.3.2 // indirect github.com/moby/sys/mountinfo v0.6.1 // indirect @@ -76,8 +76,8 @@ require ( github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect github.com/sirupsen/logrus v1.8.1 // indirect @@ -106,7 +106,6 @@ require ( golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 90996f3..960fda0 100644 --- a/go.sum +++ b/go.sum @@ -463,8 +463,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -597,8 +598,8 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -627,8 +628,9 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= @@ -776,14 +778,16 @@ github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNk github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= +github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -793,8 +797,9 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -807,8 +812,9 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1434,8 +1440,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1445,7 +1452,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= From adb95642d462cdb48303bac75a00f18f0396dc85 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 16 May 2023 17:35:58 +0300 Subject: [PATCH 384/548] [#1] Use FrostFS AIO image in integration test Signed-off-by: Alex Vanin --- integration_test.go | 56 ++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/integration_test.go b/integration_test.go index 7a170a3..549a09d 100644 --- a/integration_test.go +++ b/integration_test.go @@ -37,21 +37,16 @@ type putResponse struct { } const ( - testContainerName = "friendly" - versionWithNativeNames = "0.27.5" - testListenAddress = "localhost:8082" - testHost = "http://" + testListenAddress + testContainerName = "friendly" + testListenAddress = "localhost:8082" + testHost = "http://" + testListenAddress ) func TestIntegration(t *testing.T) { rootCtx := context.Background() - aioImage := "nspccdev/neofs-aio-testcontainer:" + aioImage := "truecloudlab/frostfs-aio:" versions := []string{ - "0.29.0", - "0.30.0", - "0.32.0", - "0.34.0", - "latest", + "1.2.5", // frostfs-storage v0.36.0 RC } key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) @@ -97,10 +92,8 @@ func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID, vers url := testHost + "/upload/" + CID.String() makePutRequestAndCheck(ctx, t, p, CID, url) - if version >= versionWithNativeNames { - url = testHost + "/upload/" + testContainerName - makePutRequestAndCheck(ctx, t, p, CID, url) - } + url = testHost + "/upload/" + testContainerName + makePutRequestAndCheck(ctx, t, p, CID, url) } func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID cid.ID, url string) { @@ -223,11 +216,9 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID require.NoError(t, err) checkGetResponse(t, resp, content, attributes) - if version >= versionWithNativeNames { - resp, err = http.Get(testHost + "/get/" + testContainerName + "/" + id.String()) - require.NoError(t, err) - checkGetResponse(t, resp, content, attributes) - } + resp, err = http.Get(testHost + "/get/" + testContainerName + "/" + id.String()) + require.NoError(t, err) + checkGetResponse(t, resp, content, attributes) } func checkGetResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) { @@ -277,11 +268,9 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID require.NoError(t, err) checkGetByAttrResponse(t, resp, content, expectedAttr) - if version >= versionWithNativeNames { - resp, err = http.Get(testHost + "/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr) - require.NoError(t, err) - checkGetByAttrResponse(t, resp, content, expectedAttr) - } + resp, err = http.Get(testHost + "/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr) + require.NoError(t, err) + checkGetByAttrResponse(t, resp, content, expectedAttr) } func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { @@ -296,10 +285,8 @@ func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID us baseURL := testHost + "/zip/" + CID.String() makeZipTest(t, baseURL, names, contents) - if version >= versionWithNativeNames { - baseURL = testHost + "/zip/" + testContainerName - makeZipTest(t, baseURL, names, contents) - } + baseURL = testHost + "/zip/" + testContainerName + makeZipTest(t, baseURL, names, contents) } func makeZipTest(t *testing.T, baseURL string, names, contents []string) { @@ -376,6 +363,7 @@ func getDefaultConfig() *viper.Viper { v.SetDefault(cfgRPCEndpoint, "http://localhost:30333") v.SetDefault("server.0.address", testListenAddress) + v.SetDefault(cfgTreeServiceEndpoint, "localhost:8080") return v } @@ -407,15 +395,11 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o container.SetCreationTime(&cnr, time.Now()) - if version >= versionWithNativeNames { - var domain container.Domain - domain.SetName(testContainerName) + var domain container.Domain + domain.SetName(testContainerName) - // currently node in aio image knows nothing about new sys attributes - // todo (@dkirillov): #2 use frostfs aio images that supports new attributes - cnr.SetAttribute(containerv2.SysAttributeNameNeoFS, domain.Name()) - cnr.SetAttribute(containerv2.SysAttributeZoneNeoFS, domain.Zone()) - } + cnr.SetAttribute(containerv2.SysAttributeName, domain.Name()) + cnr.SetAttribute(containerv2.SysAttributeZone, domain.Zone()) var waitPrm pool.WaitParams waitPrm.SetTimeout(15 * time.Second) From 1f702ad2d882e2ef0c1bf5c88b44395afa74ac06 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 16 May 2023 17:40:31 +0300 Subject: [PATCH 385/548] [#1] Update comments Signed-off-by: Alex Vanin --- uploader/multipart.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uploader/multipart.go b/uploader/multipart.go index fb6be22..cda4b34 100644 --- a/uploader/multipart.go +++ b/uploader/multipart.go @@ -16,7 +16,8 @@ type MultipartFile interface { func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartFile, error) { // To have a custom buffer (3mb) the custom multipart reader is used. - // https://github.com/nspcc-dev/neofs-http-gw/issues/148 + // Default reader uses 4KiB chunks, which slow down upload speed up to 400% + // https://github.com/golang/go/blob/91b9915d3f6f8cd2e9e9fda63f67772803adfa03/src/mime/multipart/multipart.go#L32 reader := multipart.NewReader(r, boundary) for { From 1776db289c54088ef4ba6bdec26a7c5f2294e790 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 23 May 2023 17:47:46 +0300 Subject: [PATCH 386/548] [#50] go.mod: Update min go version to 1.19 Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + go.mod | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3995307..ddb5845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ This document outlines major changes between releases. ### Changed - Update prometheus to v1.15.0 (#35) - Update go version to 1.18 (TrueCloudLab#9) +- Update go version to 1.19 (#50) - Update neo-go to v0.101.0 (#8) - Update viper to v1.15.0 (#8) - Errors have become more detailed (#18) diff --git a/go.mod b/go.mod index 20c669d..4532b1b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw -go 1.18 +go 1.19 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 From 8a229913260fe37cdb67b45197cdd8f96e61d437 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Wed, 24 May 2023 12:19:15 +0300 Subject: [PATCH 387/548] [#44] add tracing support for upload Signed-off-by: Pavel Pogodaev --- downloader/download.go | 67 +++++++++++++++++++++++++++--------------- downloader/head.go | 10 +++---- uploader/upload.go | 53 ++++++++++++++++++++------------- 3 files changed, 81 insertions(+), 49 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index b7c242b..8a62dff 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -302,43 +302,54 @@ func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(context.Context, r // byBucketname is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (d *Downloader) byBucketname(c *fasthttp.RequestCtx, f func(context.Context, request, *pool.Pool, oid.Address)) { +func (d *Downloader) byBucketname(req *fasthttp.RequestCtx, f func(context.Context, request, *pool.Pool, oid.Address)) { var ( - bucketname = c.UserValue("cid").(string) - key = c.UserValue("oid").(string) + bucketname = req.UserValue("cid").(string) + key = req.UserValue("oid").(string) log = d.log.With(zap.String("bucketname", bucketname), zap.String("key", key)) ) - cnrID, err := utils.GetContainerID(d.appCtx, bucketname, d.containerResolver) + ctx, err := tokens.StoreBearerTokenAppCtx(req, d.appCtx) if err != nil { - log.Error("wrong container id", zap.Error(err)) - response.Error(c, "wrong container id", fasthttp.StatusBadRequest) + log.Error("could not fetch and store bearer token", zap.Error(err)) + response.Error(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } - ctx, err := tokens.StoreBearerTokenAppCtx(c, d.appCtx) + ctx, span := utils.StartHTTPServerSpan(ctx, req, "GET Object by bucket name", + trace.WithAttributes( + attribute.String("bucketname", bucketname), + attribute.String("objectKey", key), + )) + defer func() { + utils.SetHTTPTraceInfo(ctx, span, req) + span.End() + }() + + cnrID, err := utils.GetContainerID(ctx, bucketname, d.containerResolver) if err != nil { - log.Error("could not fetch and store bearer token", zap.Error(err)) - response.Error(c, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) + log.Error("wrong container id", zap.Error(err)) + response.Error(req, "wrong container id", fasthttp.StatusBadRequest) return } foundOid, err := d.tree.GetLatestVersion(ctx, cnrID, key) if err != nil { log.Error("object wasn't found", zap.Error(err)) - response.Error(c, "object wasn't found", fasthttp.StatusNotFound) + response.Error(req, "object wasn't found", fasthttp.StatusNotFound) return } if foundOid.DeleteMarker { log.Error("object was deleted") - response.Error(c, "object deleted", fasthttp.StatusNotFound) + response.Error(req, "object deleted", fasthttp.StatusNotFound) return } + var addr oid.Address addr.SetContainer(*cnrID) addr.SetObject(foundOid.OID) - f(ctx, *d.newRequest(c, log), d.pool, addr) + f(ctx, *d.newRequest(req, log), d.pool, addr) } // DownloadByAttribute handles attribute-based download requests. @@ -373,7 +384,7 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, return } - res, err := d.search(c, containerID, key, val, object.MatchStringEqual) + res, err := d.search(c, ctx, containerID, key, val, object.MatchStringEqual) if err != nil { log.Error("could not search for objects", zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) @@ -404,7 +415,7 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, f(ctx, *d.newRequest(c, log), d.pool, addrObj) } -func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) (pool.ResObjectSearch, error) { +func (d *Downloader) search(c *fasthttp.RequestCtx, ctx context.Context, cid *cid.ID, key, val string, op object.SearchMatchType) (pool.ResObjectSearch, error) { filters := object.NewSearchFilters() filters.AddRootFilter() filters.AddFilter(key, val, op) @@ -416,14 +427,14 @@ func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string prm.UseBearer(*btoken) } - return d.pool.SearchObjects(d.appCtx, prm) + return d.pool.SearchObjects(ctx, prm) } -func (d *Downloader) getContainer(cnrID cid.ID) (container.Container, error) { +func (d *Downloader) getContainer(ctx context.Context, cnrID cid.ID) (container.Container, error) { var prm pool.PrmContainerGet prm.SetContainerID(cnrID) - return d.pool.GetContainer(d.appCtx, prm) + return d.pool.GetContainer(ctx, prm) } func (d *Downloader) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { @@ -450,7 +461,17 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string)) log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) - containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver) + ctx, span := utils.StartHTTPServerSpan(d.appCtx, c, "DOWNLOAD ZIP Object", + trace.WithAttributes( + attribute.String("prefix", prefix), + attribute.String("cid", scid), + )) + defer func() { + utils.SetHTTPTraceInfo(ctx, span, c) + span.End() + }() + + containerID, err := utils.GetContainerID(ctx, scid, d.containerResolver) if err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) @@ -466,7 +487,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { // check if container exists here to be able to return 404 error, // otherwise we get this error only in object iteration step // and client get 200 OK. - if _, err = d.getContainer(*containerID); err != nil { + if _, err = d.getContainer(ctx, *containerID); err != nil { log.Error("could not check container existence", zap.Error(err)) if client.IsErrContainerNotFound(err) { response.Error(c, "Not Found", fasthttp.StatusNotFound) @@ -476,7 +497,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { return } - resSearch, err := d.search(c, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) + resSearch, err := d.search(c, ctx, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error("could not search for objects", zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) @@ -509,7 +530,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { empty = false addr.SetObject(id) - if err = d.zipObject(zipWriter, addr, btoken, bufZip); err != nil { + if err = d.zipObject(ctx, zipWriter, addr, btoken, bufZip); err != nil { log.Error("failed to add object to archive", zap.String("oid", id.EncodeToString()), zap.Error(err)) } @@ -527,14 +548,14 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { }) } -func (d *Downloader) zipObject(zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error { +func (d *Downloader) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error { var prm pool.PrmObjectGet prm.SetAddress(addr) if btoken != nil { prm.UseBearer(*btoken) } - resGet, err := d.pool.GetObject(d.appCtx, prm) + resGet, err := d.pool.GetObject(ctx, prm) if err != nil { return fmt.Errorf("get FrostFS object: %v", err) } diff --git a/downloader/head.go b/downloader/head.go index a81a275..4d2f9dd 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -7,6 +7,7 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" @@ -29,11 +30,10 @@ const ( ) func headObject(ctx context.Context, req request, clnt *pool.Pool, objectAddress oid.Address) { - ctx, span := utils.StartHTTPServerSpan(ctx, req.RequestCtx, "HEAD Object", - trace.WithAttributes( - attribute.String("cid", objectAddress.Container().EncodeToString()), - attribute.String("oid", objectAddress.Object().EncodeToString()), - )) + ctx, span := tracing.StartSpanFromContext(ctx, "HEAD Object", trace.WithAttributes( + attribute.String("cid", objectAddress.Container().EncodeToString()), + attribute.String("oid", objectAddress.Object().EncodeToString()), + )) defer func() { utils.SetHTTPTraceInfo(ctx, span, req.RequestCtx) span.End() diff --git a/uploader/upload.go b/uploader/upload.go index 4e21f08..3569a0e 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -18,6 +18,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/valyala/fasthttp" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/atomic" "go.uber.org/zap" ) @@ -64,27 +66,36 @@ func New(ctx context.Context, params *utils.AppParams, settings *Settings) *Uplo } // Upload handles multipart upload request. -func (u *Uploader) Upload(c *fasthttp.RequestCtx) { +func (u *Uploader) Upload(req *fasthttp.RequestCtx) { var ( file MultipartFile idObj oid.ID addr oid.Address - scid, _ = c.UserValue("cid").(string) + scid, _ = req.UserValue("cid").(string) log = u.log.With(zap.String("cid", scid)) - bodyStream = c.RequestBodyStream() + bodyStream = req.RequestBodyStream() drainBuf = make([]byte, drainBufSize) ) - if err := tokens.StoreBearerToken(c); err != nil { + if err := tokens.StoreBearerToken(req); err != nil { log.Error("could not fetch bearer token", zap.Error(err)) - response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest) + response.Error(req, "could not fetch bearer token", fasthttp.StatusBadRequest) return } - idCnr, err := utils.GetContainerID(u.appCtx, scid, u.containerResolver) + ctx, span := utils.StartHTTPServerSpan(u.appCtx, req, "UPLOAD Object", + trace.WithAttributes( + attribute.String("cid", scid), + )) + defer func() { + utils.SetHTTPTraceInfo(ctx, span, req) + span.End() + }() + + idCnr, err := utils.GetContainerID(ctx, scid, u.containerResolver) if err != nil { log.Error("wrong container id", zap.Error(err)) - response.Error(c, "wrong container id", fasthttp.StatusBadRequest) + response.Error(req, "wrong container id", fasthttp.StatusBadRequest) return } @@ -101,21 +112,21 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { zap.Error(err), ) }() - boundary := string(c.Request.Header.MultipartFormBoundary()) + boundary := string(req.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(u.log, bodyStream, boundary); err != nil { log.Error("could not receive multipart/form", zap.Error(err)) - response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) + response.Error(req, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } - filtered, err := filterHeaders(u.log, &c.Request.Header) + filtered, err := filterHeaders(u.log, &req.Request.Header) if err != nil { log.Error("could not process headers", zap.Error(err)) - response.Error(c, err.Error(), fasthttp.StatusBadRequest) + response.Error(req, err.Error(), fasthttp.StatusBadRequest) return } now := time.Now() - if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { + if rawHeader := req.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { log.Warn("could not parse client time", zap.String("Date header", string(rawHeader)), zap.Error(err)) } else { @@ -123,9 +134,9 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { } } - if err = utils.PrepareExpirationHeader(c, u.pool, filtered, now); err != nil { + if err = utils.PrepareExpirationHeader(req, u.pool, filtered, now); err != nil { log.Error("could not prepare expiration header", zap.Error(err)) - response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) + response.Error(req, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -151,7 +162,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) attributes = append(attributes, *timestamp) } - id, bt := u.fetchOwnerAndBearerToken(c) + id, bt := u.fetchOwnerAndBearerToken(req) obj := object.New() obj.SetContainerID(*idCnr) @@ -166,8 +177,8 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { prm.UseBearer(*bt) } - if idObj, err = u.pool.PutObject(u.appCtx, prm); err != nil { - u.handlePutFrostFSErr(c, err) + if idObj, err = u.pool.PutObject(ctx, prm); err != nil { + u.handlePutFrostFSErr(req, err) return } @@ -175,9 +186,9 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { addr.SetContainer(*idCnr) // Try to return the response, otherwise, if something went wrong, throw an error. - if err = newPutResponse(addr).encode(c); err != nil { + if err = newPutResponse(addr).encode(req); err != nil { log.Error("could not encode response", zap.Error(err)) - response.Error(c, "could not encode response", fasthttp.StatusBadRequest) + response.Error(req, "could not encode response", fasthttp.StatusBadRequest) return } @@ -194,8 +205,8 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { } } // Report status code and content type. - c.Response.SetStatusCode(fasthttp.StatusOK) - c.Response.Header.SetContentType(jsonHeader) + req.Response.SetStatusCode(fasthttp.StatusOK) + req.Response.Header.SetContentType(jsonHeader) } func (u *Uploader) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) { From cdaab4feabb84a7fc97537fcd1599cca8c4e4c16 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Tue, 30 May 2023 17:01:20 +0300 Subject: [PATCH 388/548] [#44] add tracing support refactoring Signed-off-by: Pavel Pogodaev --- app.go | 62 +++++++++++++++++++++-------- downloader/download.go | 77 ++++++------------------------------- downloader/head.go | 21 +--------- main.go | 2 +- tokens/bearer-token.go | 12 ------ tokens/bearer-token_test.go | 6 ++- uploader/upload.go | 23 ++--------- utils/tracing.go | 1 + utils/util.go | 11 ++++++ 9 files changed, 78 insertions(+), 137 deletions(-) diff --git a/app.go b/app.go index a989c90..6a52a55 100644 --- a/app.go +++ b/app.go @@ -17,6 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" @@ -37,6 +38,7 @@ import ( type ( app struct { + ctx context.Context log *zap.Logger logLevel zap.AtomicLevel pool *pool.Pool @@ -60,7 +62,7 @@ type ( // App is an interface for the main gateway function. App interface { Wait() - Serve(context.Context) + Serve() } // Option is an application option. @@ -101,6 +103,7 @@ func newApp(ctx context.Context, opt ...Option) App { ) a := &app{ + ctx: ctx, log: zap.L(), cfg: viper.GetViper(), webServer: new(fasthttp.Server), @@ -353,16 +356,16 @@ func (a *app) setHealthStatus() { a.metrics.SetHealth(metrics.HealthStatusReady) } -func (a *app) Serve(ctx context.Context) { - treeClient := a.initTree(ctx) - uploadRoutes := uploader.New(ctx, a.AppParams(), a.settings.Uploader) - downloadRoutes := downloader.New(ctx, a.AppParams(), a.settings.Downloader, treeClient) +func (a *app) Serve() { + treeClient := a.initTree(a.ctx) + uploadRoutes := uploader.New(a.AppParams(), a.settings.Uploader) + downloadRoutes := downloader.New(a.AppParams(), a.settings.Downloader, treeClient) // Configure router. a.configureRouter(uploadRoutes, downloadRoutes) a.startServices() - a.initServers(ctx) + a.initServers(a.ctx) for i := range a.servers { go func(i int) { @@ -379,10 +382,10 @@ func (a *app) Serve(ctx context.Context) { LOOP: for { select { - case <-ctx.Done(): + case <-a.ctx.Done(): break LOOP case <-sigs: - a.configReload(ctx) + a.configReload(a.ctx) } } @@ -477,28 +480,55 @@ func (a *app) configureRouter(uploadRoutes *uploader.Uploader, downloadRoutes *d r.MethodNotAllowed = func(r *fasthttp.RequestCtx) { response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.logger(uploadRoutes.Upload)) + r.POST("/upload/{cid}", a.logger(a.tokenizer(a.tracer(uploadRoutes.Upload)))) a.log.Info("added path /upload/{cid}") - r.GET("/get/{cid}/{oid:*}", a.logger(downloadRoutes.DownloadByAddressOrBucketName)) - r.HEAD("/get/{cid}/{oid:*}", a.logger(downloadRoutes.HeadByAddressOrBucketName)) + r.GET("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.DownloadByAddressOrBucketName)))) + r.HEAD("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.HeadByAddressOrBucketName)))) a.log.Info("added path /get/{cid}/{oid}") - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.DownloadByAttribute)) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.HeadByAttribute)) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.DownloadByAttribute)))) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.HeadByAttribute)))) a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") - r.GET("/zip/{cid}/{prefix:*}", a.logger(downloadRoutes.DownloadZipped)) + r.GET("/zip/{cid}/{prefix:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.DownloadZipped)))) a.log.Info("added path /zip/{cid}/{prefix}") a.webServer.Handler = r.Handler } -func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { +func (a *app) logger(req fasthttp.RequestHandler) fasthttp.RequestHandler { return func(ctx *fasthttp.RequestCtx) { a.log.Info("request", zap.String("remote", ctx.RemoteAddr().String()), zap.ByteString("method", ctx.Method()), zap.ByteString("path", ctx.Path()), zap.ByteString("query", ctx.QueryArgs().QueryString()), zap.Uint64("id", ctx.ID())) - h(ctx) + req(ctx) + } +} + +func (a *app) tokenizer(req fasthttp.RequestHandler) fasthttp.RequestHandler { + return func(ctx *fasthttp.RequestCtx) { + appCtx, err := tokens.StoreBearerTokenAppCtx(ctx, a.ctx) + if err != nil { + a.log.Error("could not fetch and store bearer token", zap.Error(err)) + response.Error(ctx, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) + } + utils.SetContextToRequest(appCtx, ctx) + req(ctx) + } +} + +func (a *app) tracer(req fasthttp.RequestHandler) fasthttp.RequestHandler { + return func(ctx *fasthttp.RequestCtx) { + appCtx := utils.GetContextFromRequest(ctx) + + appCtx, span := utils.StartHTTPServerSpan(appCtx, ctx, "REQUEST") + defer func() { + utils.SetHTTPTraceInfo(appCtx, span, ctx) + span.End() + }() + + utils.SetContextToRequest(appCtx, ctx) + req(ctx) } } diff --git a/downloader/download.go b/downloader/download.go index 8a62dff..2a08c82 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -28,8 +28,6 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "go.uber.org/atomic" "go.uber.org/zap" ) @@ -97,15 +95,10 @@ func receiveFile(ctx context.Context, req request, clnt *pool.Pool, objectAddres start = time.Now() filename string ) - if err = tokens.StoreBearerToken(req.RequestCtx); err != nil { - req.log.Error("could not fetch and store bearer token", zap.Error(err)) - response.Error(req.RequestCtx, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) - return - } var prm pool.PrmObjectGet prm.SetAddress(objectAddress) - if btoken := bearerToken(req.RequestCtx); btoken != nil { + if btoken := bearerToken(ctx); btoken != nil { prm.UseBearer(*btoken) } @@ -208,7 +201,6 @@ func (r *request) handleFrostFSErr(err error, start time.Time) { // Downloader is a download request handler. type Downloader struct { - appCtx context.Context log *zap.Logger pool *pool.Pool containerResolver *resolver.ContainerResolver @@ -230,9 +222,8 @@ func (s *Settings) SetZipCompression(val bool) { } // New creates an instance of Downloader using specified options. -func New(ctx context.Context, params *utils.AppParams, settings *Settings, tree *tree.Tree) *Downloader { +func New(params *utils.AppParams, settings *Settings, tree *tree.Tree) *Downloader { return &Downloader{ - appCtx: ctx, log: params.Logger, pool: params.Pool, settings: settings, @@ -269,15 +260,7 @@ func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(context.Context, r log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj)) ) - ctx, span := utils.StartHTTPServerSpan(d.appCtx, c, "GET Object", - trace.WithAttributes( - attribute.String("cid", idCnr), - attribute.String("oid", idObj), - )) - defer func() { - utils.SetHTTPTraceInfo(ctx, span, c) - span.End() - }() + ctx := utils.GetContextFromRequest(c) cnrID, err := utils.GetContainerID(ctx, idCnr, d.containerResolver) if err != nil { @@ -309,22 +292,7 @@ func (d *Downloader) byBucketname(req *fasthttp.RequestCtx, f func(context.Conte log = d.log.With(zap.String("bucketname", bucketname), zap.String("key", key)) ) - ctx, err := tokens.StoreBearerTokenAppCtx(req, d.appCtx) - if err != nil { - log.Error("could not fetch and store bearer token", zap.Error(err)) - response.Error(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) - return - } - - ctx, span := utils.StartHTTPServerSpan(ctx, req, "GET Object by bucket name", - trace.WithAttributes( - attribute.String("bucketname", bucketname), - attribute.String("objectKey", key), - )) - defer func() { - utils.SetHTTPTraceInfo(ctx, span, req) - span.End() - }() + ctx := utils.GetContextFromRequest(req) cnrID, err := utils.GetContainerID(ctx, bucketname, d.containerResolver) if err != nil { @@ -366,16 +334,7 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ) - ctx, span := utils.StartHTTPServerSpan(d.appCtx, c, "GET Object", - trace.WithAttributes( - attribute.String("attr_key", key), - attribute.String("attr_val", val), - attribute.String("cid", scid), - )) - defer func() { - utils.SetHTTPTraceInfo(ctx, span, c) - span.End() - }() + ctx := utils.GetContextFromRequest(c) containerID, err := utils.GetContainerID(ctx, scid, d.containerResolver) if err != nil { @@ -384,7 +343,7 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, return } - res, err := d.search(c, ctx, containerID, key, val, object.MatchStringEqual) + res, err := d.search(ctx, containerID, key, val, object.MatchStringEqual) if err != nil { log.Error("could not search for objects", zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) @@ -415,7 +374,7 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, f(ctx, *d.newRequest(c, log), d.pool, addrObj) } -func (d *Downloader) search(c *fasthttp.RequestCtx, ctx context.Context, cid *cid.ID, key, val string, op object.SearchMatchType) (pool.ResObjectSearch, error) { +func (d *Downloader) search(ctx context.Context, cid *cid.ID, key, val string, op object.SearchMatchType) (pool.ResObjectSearch, error) { filters := object.NewSearchFilters() filters.AddRootFilter() filters.AddFilter(key, val, op) @@ -423,7 +382,7 @@ func (d *Downloader) search(c *fasthttp.RequestCtx, ctx context.Context, cid *ci var prm pool.PrmObjectSearch prm.SetContainerID(*cid) prm.SetFilters(filters) - if btoken := bearerToken(c); btoken != nil { + if btoken := bearerToken(ctx); btoken != nil { prm.UseBearer(*btoken) } @@ -461,15 +420,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string)) log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) - ctx, span := utils.StartHTTPServerSpan(d.appCtx, c, "DOWNLOAD ZIP Object", - trace.WithAttributes( - attribute.String("prefix", prefix), - attribute.String("cid", scid), - )) - defer func() { - utils.SetHTTPTraceInfo(ctx, span, c) - span.End() - }() + ctx := utils.GetContextFromRequest(c) containerID, err := utils.GetContainerID(ctx, scid, d.containerResolver) if err != nil { @@ -478,12 +429,6 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { return } - if err = tokens.StoreBearerToken(c); err != nil { - log.Error("could not fetch and store bearer token", zap.Error(err)) - response.Error(c, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) - return - } - // check if container exists here to be able to return 404 error, // otherwise we get this error only in object iteration step // and client get 200 OK. @@ -497,7 +442,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { return } - resSearch, err := d.search(c, ctx, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) + resSearch, err := d.search(ctx, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error("could not search for objects", zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) @@ -518,7 +463,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { empty := true called := false - btoken := bearerToken(c) + btoken := bearerToken(ctx) addr.SetContainer(*containerID) errIter := resSearch.Iterate(func(id oid.ID) bool { diff --git a/downloader/head.go b/downloader/head.go index 4d2f9dd..3e5a92a 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -7,16 +7,11 @@ import ( "strconv" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -30,23 +25,9 @@ const ( ) func headObject(ctx context.Context, req request, clnt *pool.Pool, objectAddress oid.Address) { - ctx, span := tracing.StartSpanFromContext(ctx, "HEAD Object", trace.WithAttributes( - attribute.String("cid", objectAddress.Container().EncodeToString()), - attribute.String("oid", objectAddress.Object().EncodeToString()), - )) - defer func() { - utils.SetHTTPTraceInfo(ctx, span, req.RequestCtx) - span.End() - }() - var start = time.Now() - if err := tokens.StoreBearerToken(req.RequestCtx); err != nil { - req.log.Error("could not fetch and store bearer token", zap.Error(err)) - response.Error(req.RequestCtx, "could not fetch and store bearer token", fasthttp.StatusBadRequest) - return - } - btoken := bearerToken(req.RequestCtx) + btoken := bearerToken(ctx) var prm pool.PrmObjectHead prm.SetAddress(objectAddress) diff --git a/main.go b/main.go index f997955..5762675 100644 --- a/main.go +++ b/main.go @@ -12,6 +12,6 @@ func main() { logger, atomicLevel := newLogger(v) application := newApp(globalContext, WithLogger(logger, atomicLevel), WithConfig(v)) - go application.Serve(globalContext) + go application.Serve() application.Wait() } diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index f43cb85..fd4404b 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -48,18 +48,6 @@ func BearerTokenFromCookie(h *fasthttp.RequestHeader) []byte { return auth } -// StoreBearerToken extracts a bearer token from the header or cookie and stores -// it in the request context. -func StoreBearerToken(ctx *fasthttp.RequestCtx) error { - tkn, err := fetchBearerToken(ctx) - if err != nil { - return err - } - // This is an analog of context.WithValue. - ctx.SetUserValue(bearerTokenKey, tkn) - return nil -} - // StoreBearerTokenAppCtx extracts a bearer token from the header or cookie and stores // it in the application context. func StoreBearerTokenAppCtx(ctx *fasthttp.RequestCtx, appCtx context.Context) (context.Context, error) { diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index d5706cc..170246c 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -3,6 +3,7 @@ package tokens import ( + "context" "encoding/base64" "testing" @@ -153,10 +154,11 @@ func Test_checkAndPropagateBearerToken(t *testing.T) { ctx := makeTestRequest(t64, "") // Expect to see the token within the context. - require.NoError(t, StoreBearerToken(ctx)) + appCtx, err := StoreBearerTokenAppCtx(ctx, context.Background()) + require.NoError(t, err) // Expect to see the same token without errors. - actual, err := LoadBearerToken(ctx) + actual, err := LoadBearerToken(appCtx) require.NoError(t, err) require.Equal(t, tkn, actual) } diff --git a/uploader/upload.go b/uploader/upload.go index 3569a0e..43996ea 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -18,8 +18,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/valyala/fasthttp" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "go.uber.org/atomic" "go.uber.org/zap" ) @@ -31,7 +29,6 @@ const ( // Uploader is an upload request handler. type Uploader struct { - appCtx context.Context log *zap.Logger pool *pool.Pool ownerID *user.ID @@ -54,9 +51,8 @@ func (s *Settings) SetDefaultTimestamp(val bool) { // New creates a new Uploader using specified logger, connection pool and // other options. -func New(ctx context.Context, params *utils.AppParams, settings *Settings) *Uploader { +func New(params *utils.AppParams, settings *Settings) *Uploader { return &Uploader{ - appCtx: ctx, log: params.Logger, pool: params.Pool, ownerID: params.Owner, @@ -77,20 +73,7 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { drainBuf = make([]byte, drainBufSize) ) - if err := tokens.StoreBearerToken(req); err != nil { - log.Error("could not fetch bearer token", zap.Error(err)) - response.Error(req, "could not fetch bearer token", fasthttp.StatusBadRequest) - return - } - - ctx, span := utils.StartHTTPServerSpan(u.appCtx, req, "UPLOAD Object", - trace.WithAttributes( - attribute.String("cid", scid), - )) - defer func() { - utils.SetHTTPTraceInfo(ctx, span, req) - span.End() - }() + ctx := utils.GetContextFromRequest(req) idCnr, err := utils.GetContainerID(ctx, scid, u.containerResolver) if err != nil { @@ -162,7 +145,7 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) attributes = append(attributes, *timestamp) } - id, bt := u.fetchOwnerAndBearerToken(req) + id, bt := u.fetchOwnerAndBearerToken(ctx) obj := object.New() obj.SetContainerID(*idCnr) diff --git a/utils/tracing.go b/utils/tracing.go index 4bb74df..75a4486 100644 --- a/utils/tracing.go +++ b/utils/tracing.go @@ -77,6 +77,7 @@ func StartHTTPServerSpan(ctx context.Context, req *fasthttp.RequestCtx, operatio attribute.String("http.path", string(req.Path())), semconv.HTTPMethod(string(req.Method())), semconv.RPCService("frostfs-http-gw"), + attribute.String("http.query", req.QueryArgs().String()), ), trace.WithSpanKind(trace.SpanKindServer)) return tracing.StartSpanFromContext(ctx, operationName, opts...) } diff --git a/utils/util.go b/utils/util.go index d5a476a..e54c98d 100644 --- a/utils/util.go +++ b/utils/util.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + "github.com/valyala/fasthttp" ) // GetContainerID decode container id, if it's not a valid container id @@ -43,3 +44,13 @@ func GetEpochDurations(ctx context.Context, p *pool.Pool) (*EpochDurations, erro } return res, nil } + +// SetContextToRequest adds new context to fasthttp request. +func SetContextToRequest(ctx context.Context, c *fasthttp.RequestCtx) { + c.SetUserValue("context", ctx) +} + +// GetContextFromRequest returns main context from fasthttp request context. +func GetContextFromRequest(c *fasthttp.RequestCtx) context.Context { + return c.UserValue("context").(context.Context) +} From c4fe718556da57a9975278a2363120ef7b92c92f Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 31 May 2023 16:04:31 +0300 Subject: [PATCH 389/548] [#54] Move files from .github to .forgejo Signed-off-by: Alex Vanin --- {.github => .forgejo}/CODEOWNERS | 0 {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md | 0 {.github => .forgejo}/ISSUE_TEMPLATE/config.yml | 0 {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md | 0 {.github => .forgejo}/logo.svg | 0 {.github => .forgejo}/workflows/builds.yml | 0 {.github => .forgejo}/workflows/codeql-analysis.yml | 0 {.github => .forgejo}/workflows/dco.yml | 0 {.github => .forgejo}/workflows/tests.yml | 0 README.md | 2 +- 10 files changed, 1 insertion(+), 1 deletion(-) rename {.github => .forgejo}/CODEOWNERS (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/config.yml (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md (100%) rename {.github => .forgejo}/logo.svg (100%) rename {.github => .forgejo}/workflows/builds.yml (100%) rename {.github => .forgejo}/workflows/codeql-analysis.yml (100%) rename {.github => .forgejo}/workflows/dco.yml (100%) rename {.github => .forgejo}/workflows/tests.yml (100%) diff --git a/.github/CODEOWNERS b/.forgejo/CODEOWNERS similarity index 100% rename from .github/CODEOWNERS rename to .forgejo/CODEOWNERS diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.forgejo/ISSUE_TEMPLATE/bug_report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.md rename to .forgejo/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.forgejo/ISSUE_TEMPLATE/config.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/config.yml rename to .forgejo/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.forgejo/ISSUE_TEMPLATE/feature_request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .forgejo/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/logo.svg b/.forgejo/logo.svg similarity index 100% rename from .github/logo.svg rename to .forgejo/logo.svg diff --git a/.github/workflows/builds.yml b/.forgejo/workflows/builds.yml similarity index 100% rename from .github/workflows/builds.yml rename to .forgejo/workflows/builds.yml diff --git a/.github/workflows/codeql-analysis.yml b/.forgejo/workflows/codeql-analysis.yml similarity index 100% rename from .github/workflows/codeql-analysis.yml rename to .forgejo/workflows/codeql-analysis.yml diff --git a/.github/workflows/dco.yml b/.forgejo/workflows/dco.yml similarity index 100% rename from .github/workflows/dco.yml rename to .forgejo/workflows/dco.yml diff --git a/.github/workflows/tests.yml b/.forgejo/workflows/tests.yml similarity index 100% rename from .github/workflows/tests.yml rename to .forgejo/workflows/tests.yml diff --git a/README.md b/README.md index 1e050af..e2b697a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

-FrostFS logo +FrostFS logo

FrostFS is a decentralized distributed object storage integrated with the NEO Blockchain. From 01b9df83e6188a06cccc031e7d042a752a2ec6ce Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 31 May 2023 16:06:00 +0300 Subject: [PATCH 390/548] [#54] Update actions for forgejo ci runner Signed-off-by: Alex Vanin --- .forgejo/workflows/builds.yml | 73 +++++-------------------- .forgejo/workflows/codeql-analysis.yml | 67 ----------------------- .forgejo/workflows/dco.yml | 34 ++++++------ .forgejo/workflows/tests.yml | 76 +++++--------------------- 4 files changed, 44 insertions(+), 206 deletions(-) delete mode 100644 .forgejo/workflows/codeql-analysis.yml diff --git a/.forgejo/workflows/builds.yml b/.forgejo/workflows/builds.yml index 64e2e5e..3df5081 100644 --- a/.forgejo/workflows/builds.yml +++ b/.forgejo/workflows/builds.yml @@ -1,67 +1,20 @@ -name: Builds - -on: - pull_request: - branches: - - master - - 'support/*' - types: [opened, synchronize] - paths-ignore: - - '**/*.md' +on: [pull_request] jobs: - build_cli: - name: Build CLI - runs-on: ubuntu-20.04 - + builds: + name: Builds + runs-on: ubuntu-latest + strategy: + matrix: + go_versions: [ '1.19', '1.20' ] + fail-fast: false steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Setup Go - uses: actions/setup-go@v2 - with: - go-version: 1.19 - - - name: Restore Go modules from cache - uses: actions/cache@v2 - with: - path: /home/runner/go/pkg/mod - key: deps-${{ hashFiles('go.sum') }} - - - name: Update Go modules - run: make dep - - - name: Build CLI - run: make - - - name: Check version - run: if [[ $(make version) == *"dirty"* ]]; then exit 1; fi - - build_image: - needs: build_cli - name: Build Docker image - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - uses: actions/checkout@v3 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: 1.19 + go-version: '${{ matrix.go_versions }}' - - name: Restore Go modules from cache - uses: actions/cache@v2 - with: - path: /home/runner/go/pkg/mod - key: deps-${{ hashFiles('go.sum') }} - - - name: Update Go modules - run: make dep - - - name: Build Docker image - run: make image + - name: Build binary + run: make diff --git a/.forgejo/workflows/codeql-analysis.yml b/.forgejo/workflows/codeql-analysis.yml deleted file mode 100644 index 8598516..0000000 --- a/.forgejo/workflows/codeql-analysis.yml +++ /dev/null @@ -1,67 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ master, 'support/*' ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master, 'support/*' ] - schedule: - - cron: '35 8 * * 1' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 397e961..b1ccd0e 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -1,22 +1,20 @@ -name: DCO check - -on: - pull_request: - branches: - - master - - 'support/*' +on: [pull_request] jobs: - commits_check_job: + dco: + name: DCO runs-on: ubuntu-latest - name: Commits Check steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@master - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@master - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + + - name: Run commit format checker + uses: https://git.alexvan.in/alexvanin/dco-go@v1 + with: + from: adb95642d diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index e4c210f..f64b816 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -1,86 +1,40 @@ -name: Tests - -on: - pull_request: - branches: - - master - - 'support/*' - types: [opened, synchronize] - paths-ignore: - - '**/*.md' +on: [pull_request] jobs: lint: name: Lint runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + + - name: Sync tree service + run: make sync-tree + - name: golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: https://github.com/golangci/golangci-lint-action@v2 with: version: latest - cover: - name: Coverage - runs-on: ubuntu-20.04 - - env: - CGO_ENABLED: 1 - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.19 - - - name: Restore Go modules from cache - uses: actions/cache@v2 - with: - path: /home/runner/go/pkg/mod - key: deps-${{ hashFiles('go.sum') }} - - - name: Update Go modules - run: make dep - - - name: Test and write coverage profile - run: make cover - - - name: Upload coverage results to Codecov - uses: codecov/codecov-action@v1 - with: - fail_ci_if_error: false - path_to_write_report: ./coverage.txt - verbose: true - tests: name: Tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.18', '1.19' ] + go_versions: [ '1.19', '1.20' ] fail-fast: false steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - uses: actions/checkout@v3 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: '${{ matrix.go_versions }}' - - - name: Restore Go modules from cache - uses: actions/cache@v2 - with: - path: /home/runner/go/pkg/mod - key: deps-${{ hashFiles('go.sum') }} + + - name: Sync tree service + run: make sync-tree - name: Update Go modules run: make dep - name: Run tests - run: make test + run: make test \ No newline at end of file From f17f6747c44a6493a478c23768dce052429f9a0b Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 31 May 2023 16:34:04 +0300 Subject: [PATCH 391/548] [#54] Fix linter warnings Signed-off-by: Alex Vanin --- app.go | 42 ++++++++++++++++++------------------- settings.go | 6 +----- tokens/bearer-token.go | 12 ++++++----- tokens/bearer-token_test.go | 4 ++-- 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/app.go b/app.go index 6a52a55..f43d187 100644 --- a/app.go +++ b/app.go @@ -494,41 +494,41 @@ func (a *app) configureRouter(uploadRoutes *uploader.Uploader, downloadRoutes *d a.webServer.Handler = r.Handler } -func (a *app) logger(req fasthttp.RequestHandler) fasthttp.RequestHandler { - return func(ctx *fasthttp.RequestCtx) { - a.log.Info("request", zap.String("remote", ctx.RemoteAddr().String()), - zap.ByteString("method", ctx.Method()), - zap.ByteString("path", ctx.Path()), - zap.ByteString("query", ctx.QueryArgs().QueryString()), - zap.Uint64("id", ctx.ID())) - req(ctx) +func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { + return func(req *fasthttp.RequestCtx) { + a.log.Info("request", zap.String("remote", req.RemoteAddr().String()), + zap.ByteString("method", req.Method()), + zap.ByteString("path", req.Path()), + zap.ByteString("query", req.QueryArgs().QueryString()), + zap.Uint64("id", req.ID())) + h(req) } } -func (a *app) tokenizer(req fasthttp.RequestHandler) fasthttp.RequestHandler { - return func(ctx *fasthttp.RequestCtx) { - appCtx, err := tokens.StoreBearerTokenAppCtx(ctx, a.ctx) +func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { + return func(req *fasthttp.RequestCtx) { + appCtx, err := tokens.StoreBearerTokenAppCtx(a.ctx, req) if err != nil { a.log.Error("could not fetch and store bearer token", zap.Error(err)) - response.Error(ctx, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) + response.Error(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) } - utils.SetContextToRequest(appCtx, ctx) - req(ctx) + utils.SetContextToRequest(appCtx, req) + h(req) } } -func (a *app) tracer(req fasthttp.RequestHandler) fasthttp.RequestHandler { - return func(ctx *fasthttp.RequestCtx) { - appCtx := utils.GetContextFromRequest(ctx) +func (a *app) tracer(h fasthttp.RequestHandler) fasthttp.RequestHandler { + return func(req *fasthttp.RequestCtx) { + appCtx := utils.GetContextFromRequest(req) - appCtx, span := utils.StartHTTPServerSpan(appCtx, ctx, "REQUEST") + appCtx, span := utils.StartHTTPServerSpan(appCtx, req, "REQUEST") defer func() { - utils.SetHTTPTraceInfo(appCtx, span, ctx) + utils.SetHTTPTraceInfo(appCtx, span, req) span.End() }() - utils.SetContextToRequest(appCtx, ctx) - req(ctx) + utils.SetContextToRequest(appCtx, req) + h(req) } } diff --git a/settings.go b/settings.go index 87c56bc..7a78c53 100644 --- a/settings.go +++ b/settings.go @@ -320,11 +320,7 @@ func mergeConfig(v *viper.Viper, fileName string) error { } }() - if err = v.MergeConfig(cfgFile); err != nil { - return err - } - - return nil + return v.MergeConfig(cfgFile) } // newLogger constructs a zap.Logger instance for current application. diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index fd4404b..b01860d 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -13,9 +13,11 @@ import ( type fromHandler = func(h *fasthttp.RequestHeader) []byte +type ctxKey string + const ( - bearerTokenHdr = "Bearer" - bearerTokenKey = "__context_bearer_token_key" + bearerTokenHdr = "Bearer" + bearerTokenKey ctxKey = "__context_bearer_token_key" ) // BearerToken usage: @@ -50,12 +52,12 @@ func BearerTokenFromCookie(h *fasthttp.RequestHeader) []byte { // StoreBearerTokenAppCtx extracts a bearer token from the header or cookie and stores // it in the application context. -func StoreBearerTokenAppCtx(ctx *fasthttp.RequestCtx, appCtx context.Context) (context.Context, error) { - tkn, err := fetchBearerToken(ctx) +func StoreBearerTokenAppCtx(ctx context.Context, req *fasthttp.RequestCtx) (context.Context, error) { + tkn, err := fetchBearerToken(req) if err != nil { return nil, err } - newCtx := context.WithValue(appCtx, bearerTokenKey, tkn) + newCtx := context.WithValue(ctx, bearerTokenKey, tkn) return newCtx, nil } diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index 170246c..cc54e74 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -151,10 +151,10 @@ func Test_checkAndPropagateBearerToken(t *testing.T) { t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) require.NotEmpty(t, t64) - ctx := makeTestRequest(t64, "") + req := makeTestRequest(t64, "") // Expect to see the token within the context. - appCtx, err := StoreBearerTokenAppCtx(ctx, context.Background()) + appCtx, err := StoreBearerTokenAppCtx(context.Background(), req) require.NoError(t, err) // Expect to see the same token without errors. From f24f39ec92c05e19b7eab165beef3844435c5092 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 5 Jun 2023 13:03:21 +0300 Subject: [PATCH 392/548] [#56] Increase golangci-lint timeout For slow actions runners. Signed-off-by: Alex Vanin --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 67f93e7..a271450 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -4,7 +4,7 @@ # options for analysis running run: # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 5m + timeout: 15m # include test files or not, default is true tests: true From b8944adb65a7d87e6379b592fd7f6134fddc62ae Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 6 Jun 2023 17:28:52 +0300 Subject: [PATCH 393/548] [#55] Add govulncheck in CI Check dependency issues on every PR. Signed-off-by: Alex Vanin --- .forgejo/workflows/vulncheck.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .forgejo/workflows/vulncheck.yml diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml new file mode 100644 index 0000000..34692c9 --- /dev/null +++ b/.forgejo/workflows/vulncheck.yml @@ -0,0 +1,24 @@ +on: [pull_request] + +jobs: + vulncheck: + name: Vulncheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Sync tree service + run: make sync-tree + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + + - name: Install govulncheck + run: go install golang.org/x/vuln/cmd/govulncheck@latest + + - name: Run govulncheck + run: govulncheck ./... From 9765adf844644896dd43361c582586b904c70a41 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 6 Jun 2023 15:30:32 +0300 Subject: [PATCH 394/548] [#2] Update CODEOWNERS Codeownders feature isn't supported yet in forgejo nor gitea. Looking for github.com/go-gitea/gitea/pull/24910 Signed-off-by: Alex Vanin --- .forgejo/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/CODEOWNERS b/.forgejo/CODEOWNERS index 37417c9..c280648 100644 --- a/.forgejo/CODEOWNERS +++ b/.forgejo/CODEOWNERS @@ -1 +1 @@ -* @alexvanin @KirillovDenis +* @alexvanin @dkirillov From 61d152ee6a3307bc6c8fcb4beef779456bb58c01 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 6 Jun 2023 15:43:41 +0300 Subject: [PATCH 395/548] [#2] Update CHANGELOG Replace changelog history before the fork with the link to the fork source. Signed-off-by: Alex Vanin --- CHANGELOG.md | 274 +-------------------------------------------------- 1 file changed, 4 insertions(+), 270 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddb5845..11d43e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ This document outlines major changes between releases. - Separate integration tests with build tags (#24) - Changed values for `frostfs_http_gw_state_health` metric (#32) -### Updating from v0.26.0 +### Updating from neofs-http-gw v0.26.0 To set system attributes use updated headers (you can use old ones for now, but their support will be dropped in the future releases): @@ -29,276 +29,10 @@ To set system attributes use updated headers * `X-Attribute-NEOFS-*` -> `X-Attribute-SYSTEM-*` * `X-Attribute-neofs-*` -> `X-Attribute-system-*` -## [0.26.0] - 2022-12-28 - -### Fixed -- ENV config example (#236) - -### Added -- Support the `Date` header on upload (#214) -- Available routes specification (#216) -- Mention caching strategy in docs (#215) -- Add error response on attribute duplicates (#221) -- Multiple server listeners (#228) - -### Removed -- Deprecated linters (#239) - -### Updating from v0.25.1 -Make sure your configuration is valid: - -If you configure application using environment variables change: -* `HTTP_GW_LISTEN_ADDRESS` -> `HTTP_GW_SERVER_0_ADDRESS` -* `HTTP_GW_TLS_CERT_FILE` -> `HTTP_GW_SERVER_0_TLS_CERT_FILE` (and set `HTTP_GW_SERVER_0_TLS_ENABLED=true`) -* `HTTP_GW_TLS_KEY_FILE` -> `HTTP_GW_SERVER_0_TLS_KEY_FILE` (and set `HTTP_GW_SERVER_0_TLS_ENABLED=true`) - -If you configure application using `.yaml` file change: -* `listen_address` -> `server.0.address` -* `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`) -* `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`) - -## [0.25.1] - 2022-11-30 - -### Fixed -- Download zip archive when `FilePath` is invalid (#222) -- Only one peer must be healthy to init pool (#233) - -### Added -- Debian packaging (#223) -- Timeout for individual operations in streaming RPC (#234) - -## [0.25.0] - 2022-10-31 - -### Added -- Config reloading on SIGHUP (#200, #208) -- Stop pool dial on SIGINT (#212) -- Makefile help (#213) - -### Changed -- Update NeoFS error handling (#206) -- GitHub actions updates (#205, #209) -- Unified system attribute format for GET and HEAD (#213) - -## [0.24.0] - 2022-09-14 - -### Fixed -- Fix expiration epoch calculation (#198) -- Fix panic on go1.19 (#188) - -### Added -- Exposure of pool metrics (#179, #194) - -### Changed -- Help doesn't print empty parameters (#186) -- Update version calculation (#190, #199) -- Update neofs-sdk-go (#196) -- Update go version in CI and docker (#197, #202) - -## [0.23.0] - 2022-08-02 - -### Added -- New param to configure pool error threshold (#184) - -### Changed -- Pprof and prometheus metrics configuration (#171) -- Drop GO111MODULES from builds (#182) - -### Updating from v0.22.0 -1. To enable pprof use `pprof.enabled` instead of `pprof` in config. - To enable prometheus metrics use `prometheus.enabled` instead of `metrics` in config. - If you are using the command line flags you can skip this step. - -## [0.22.0] - 2022-07-25 - -### Added -- Default params documentation (#172) -- Health metric (#175) - -### Changed -- Version output (#169) -- Updated SDK Version (#178) - -## [0.21.0] - 2022-06-20 - -### Fixed -- Downloading ZIP archive using streaming (#163) - -### Added -- New make target to build app in docker (#159) - -### Changed -- Increased buffer size for file uploading (#148) -- Updated linter version to v1.46.2 (#161) -- Updated CodeQL version to v2 (#158) - - -## [0.20.0] - 2022-04-29 - -### Fixed -- Get rid of data race on server shutdown (#145) -- Improved English in docs and comments (#153) -- Use `FilePath` to download zip (#150) - -### Added -- Support container name NNS resolving (#142) - -### Changed -- Updated docs (#133, #140) -- Increased default read/write timeouts (#154) -- Updated SDK (#137, #139) -- Updated go version to 1.17 (#143) -- Improved error messages (#144) - -## [0.19.0] - 2022-03-16 - -### Fixed -- Uploading object with zero payload (#122) -- Different headers format in GET and HEAD (#125) -- Fixed project name in docs (#120) - -### Added -- Support object attributes with spaces (#123) - -### Changed -- Updated fasthttp to v1.34.0 (#129) -- Updated NeoFS SDK to v1.0.0-rc.3 (#126, #132) -- Refactored content type detecting (#128) - - -## [0.18.0] - 2021-12-10 - -### Fixed -- System headers format (#111) - -### Added -- Different formats to set object's expiration: in epoch, duration, timestamp, - RFC3339 (#108) -- Support of nodes priority (#115) - -### Changed -- Updated testcontainers dependency (#100) - -## [0.17.0] - 2021-11-15 - -Support of bulk file download with zip streams and various bug fixes. - -### Fixed -- Allow canonical `X-Attribute-Neofs-*` headers (#87) -- Responses with error message now end with `\n` character (#105) -- Application does not require all neofs endpoints to be healthy at start now - (#103) -- Application now tracks session token errors and recreates tokens in runtime - (#95) - -### Added -- Integration tests with [all-in-one](https://github.com/nspcc-dev/neofs-aio/) - test containers (#85, #94) -- Bulk download support with zip streams (#92, #96) - -## 0.16.1 (28 Jul 2021) - -New features: -* logging requests (#77) -* HEAD methods for download routes (#76) - -Improvements: -* updated sdk-go dependency (#82) - -Bugs fixed: -* wrong NotFound status was used (#30) - -## 0.16.0 (29 Jun 2021) - -We update HTTP gateway with NEP-6 wallets support, YAML configuration files -and small fixes. - -New features: - * YAML configuration file (#71) - -Behavior changes: - * gateway key needs to be stored in a proper NEP-6 wallet now, `-k` option is - no longer available, see `-w` and `-a` (#68) - -Bugs fixed: - * downloads were not streamed leading to excessive memory usage (#67) - * Last-Modified header incorrectly used local time (#75) - -## 0.15.2 (22 Jun 2021) - -New features: - * Content-Type returned for object GET requests can now be taken from - attributes (overriding autodetection, #65) - -Behavior changes: - * grpc keepalive options can no longer be changed (#60) - -Improvements: - * code refactoring (more reuse between different gateways, moved some code to - sdk-go, #47, #46, #51, #62, #63) - * documentation updates and fixes (#53, #49, #55, #59) - * updated api-go dependency (#57) - -Bugs fixed: - * `-k` option wasn't accepted for key although it was documented (#50) - -## 0.15.1 (24 May 2021) - -This important release makes HTTP gateway compatible with NeoFS node version -0.20.0. - -Behavior changes: - * neofs-api-go was updated to 1.26.1, which contains some incompatible - changes in underlying components (#39, #44) - * `neofs-http-gw` is consistently used now for repository, binary and image - names (#43) - -Improvements: - * minor code cleanups based on stricter set of linters (#41) - * updated README (#42) - -## 0.15.0 (30 Apr 2021) - -This is the first public release incorporating latest NeoFS protocol support -and fixing some bugs. - -New features: - * upload support (#14, #13, #29) - * ephemeral keys (#26) - * TLS server support (#28) - -Behavior changes: - * node weights can now be specified as simple numbers instead of percentages - and gateway will calculate the proportion automatically (#27) - * attributes are converted now to `X-Attribute-*` headers when retrieving - object from gate instead of `X-*` (#29) - -Improvements: - * better Makefile (#16, #24, #33, #34) - * updated documentation (#16, #29, #35, #36) - * updated neofs-api-go to v1.25.0 (#17, #20) - * updated fasthttp to v1.23.0+ (#17, #29) - * refactoring, eliminating some dependencies (#20, #29) - -Bugs fixed: - * gateway attempted to work with no NeoFS peers configured (#29) - * some invalid headers could be sent for attributes using non-ASCII or - non-printable characters (#29) ## Older versions -Please refer to [Github -releases](https://github.com/nspcc-dev/neofs-http-gw/releases/) for older -releases. +This project is a fork of [NeoFS HTTP Gateway](https://github.com/nspcc-dev/neofs-http-gw) from version v0.26.0. +To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs-http-gw/blob/master/CHANGELOG.md. -[0.17.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.16.1...v0.17.0 -[0.18.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.17.0...v0.18.0 -[0.19.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.18.0...v0.19.0 -[0.20.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.19.0...v0.20.0 -[0.21.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.20.0...v0.21.0 -[0.22.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.21.0...v0.22.0 -[0.23.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.22.0...v0.23.0 -[0.24.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.23.0...v0.24.0 -[0.25.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.24.0...v0.25.0 -[0.25.1]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.0...v0.25.1 -[0.26.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.1...v0.26.0 -[Unreleased]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.26.0...master +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/72734ab4...master From d7dbff12553c4943198174c8a2c719a6651e2b1e Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 6 Jun 2023 15:44:56 +0300 Subject: [PATCH 396/548] [#2] Use latest AIO image in integration test Latest version provides more stability during startup stage. Signed-off-by: Alex Vanin --- integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_test.go b/integration_test.go index 549a09d..014f923 100644 --- a/integration_test.go +++ b/integration_test.go @@ -46,7 +46,7 @@ func TestIntegration(t *testing.T) { rootCtx := context.Background() aioImage := "truecloudlab/frostfs-aio:" versions := []string{ - "1.2.5", // frostfs-storage v0.36.0 RC + "1.2.7", // frostfs-storage v0.36.0 RC } key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) From d0f6baa44bc75dc7bb8c1291add08bbc7bd4a58d Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 6 Jun 2023 16:00:31 +0300 Subject: [PATCH 397/548] [#2] Return shields Signed-off-by: Alex Vanin --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e2b697a..52506bd 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ --- [![Report](https://goreportcard.com/badge/git.frostfs.info/TrueCloudLab/frostfs-http-gw)](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-http-gw) +![Release](https://img.shields.io/badge/dynamic/json.svg?label=release&url=https://git.frostfs.info/api/v1/repos/TrueCloudLab/frostfs-http-gw/releases&query=$[0].tag_name&color=orange) +![License](https://img.shields.io/badge/license-GPL--3.0-orange.svg) # FrostFS HTTP Gateway From 2c706bec71cd5595bf7d75fd250a02d558c3d48a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 6 Jun 2023 17:15:07 +0300 Subject: [PATCH 398/548] [#2] Update CONTRIBUTING Signed-off-by: Alex Vanin --- CONTRIBUTING.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b365b64..ffd587d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,8 +3,8 @@ First, thank you for contributing! We love and encourage pull requests from everyone. Please follow the guidelines: -- Check the open [issues](https://github.com/TrueCloudLab/frostfs-http-gw/issues) and - [pull requests](https://github.com/TrueCloudLab/frostfs-http-gw/pulls) for existing +- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/issues) and + [pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/pulls) for existing discussions. - Open an issue first, to discuss a new feature or enhancement. @@ -27,20 +27,20 @@ Start by forking the `frostfs-http-gw` repository, make changes in a branch and send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: -### Set up your GitHub Repository +### Set up your git repository Fork [FrostFS HTTP Gateway -upstream](https://github.com/TrueCloudLab/frostfs-http-gw/fork) source repository +upstream](https://git.frostfs.info/repo/fork/8) source repository to your own personal repository. Copy the URL of your fork (you will need it for the `git clone` command below). ```sh -$ git clone https://github.com/TrueCloudLab/frostfs-http-gw +$ git clone https://git.frostfs.info//frostfs-http-gw.git ``` ### Set up git remote as ``upstream`` ```sh $ cd frostfs-http-gw -$ git remote add upstream https://github.com/TrueCloudLab/frostfs-http-gw +$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-http-gw.git $ git fetch upstream $ git merge upstream/master ... @@ -90,8 +90,8 @@ $ git push origin feature/123-something_awesome ``` ### Create a Pull Request -Pull requests can be created via GitHub. Refer to [this -document](https://help.github.com/articles/creating-a-pull-request/) for +Pull requests can be created via Forgejo. Refer to [this +document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged. From 5be537321bf4c74a5ba269cd60146cd4e32ff320 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 14:18:31 +0300 Subject: [PATCH 399/548] [#2] Keep issue templates in .github The only thing supported in .forgejo dir is a workflows. It might be useful to keep .github dir for repo mirrors. Signed-off-by: Alex Vanin --- {.forgejo => .github}/CODEOWNERS | 0 {.forgejo => .github}/ISSUE_TEMPLATE/bug_report.md | 0 {.forgejo => .github}/ISSUE_TEMPLATE/config.yml | 0 {.forgejo => .github}/ISSUE_TEMPLATE/feature_request.md | 0 {.forgejo => .github}/logo.svg | 0 README.md | 2 +- 6 files changed, 1 insertion(+), 1 deletion(-) rename {.forgejo => .github}/CODEOWNERS (100%) rename {.forgejo => .github}/ISSUE_TEMPLATE/bug_report.md (100%) rename {.forgejo => .github}/ISSUE_TEMPLATE/config.yml (100%) rename {.forgejo => .github}/ISSUE_TEMPLATE/feature_request.md (100%) rename {.forgejo => .github}/logo.svg (100%) diff --git a/.forgejo/CODEOWNERS b/.github/CODEOWNERS similarity index 100% rename from .forgejo/CODEOWNERS rename to .github/CODEOWNERS diff --git a/.forgejo/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md similarity index 100% rename from .forgejo/ISSUE_TEMPLATE/bug_report.md rename to .github/ISSUE_TEMPLATE/bug_report.md diff --git a/.forgejo/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml similarity index 100% rename from .forgejo/ISSUE_TEMPLATE/config.yml rename to .github/ISSUE_TEMPLATE/config.yml diff --git a/.forgejo/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md similarity index 100% rename from .forgejo/ISSUE_TEMPLATE/feature_request.md rename to .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.forgejo/logo.svg b/.github/logo.svg similarity index 100% rename from .forgejo/logo.svg rename to .github/logo.svg diff --git a/README.md b/README.md index 52506bd..6e19d31 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

-FrostFS logo +FrostFS logo

FrostFS is a decentralized distributed object storage integrated with the NEO Blockchain. From 1dfbe36eca62589b548d9c0037dd12a9015d49e6 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 9 Jun 2023 09:39:01 +0300 Subject: [PATCH 400/548] [#59] Use tree pool from SDK Signed-off-by: Denis Kirillov --- app.go | 80 ++------------ go.mod | 66 ++++++------ go.sum | 126 +++++++++++----------- integration_test.go | 1 - internal/frostfs/services/pool_wrapper.go | 115 ++++++++++++++++++++ settings.go | 114 +++++++++++++++++++- utils/tracing.go | 2 +- 7 files changed, 338 insertions(+), 166 deletions(-) create mode 100644 internal/frostfs/services/pool_wrapper.go diff --git a/app.go b/app.go index f43d187..88f9a07 100644 --- a/app.go +++ b/app.go @@ -6,12 +6,10 @@ import ( "net/http" "os" "os/signal" - "strconv" "sync" "syscall" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/downloader" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" @@ -21,7 +19,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/fasthttp/router" "github.com/nspcc-dev/neo-go/cli/flags" @@ -32,8 +32,6 @@ import ( "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) type ( @@ -42,6 +40,7 @@ type ( log *zap.Logger logLevel zap.AtomicLevel pool *pool.Pool + treePool *treepool.Pool key *keys.PrivateKey owner *user.ID cfg *viper.Viper @@ -98,10 +97,6 @@ func WithConfig(c *viper.Viper) Option { } func newApp(ctx context.Context, opt ...Option) App { - var ( - err error - ) - a := &app{ ctx: ctx, log: zap.L(), @@ -126,51 +121,12 @@ func newApp(ctx context.Context, opt ...Option) App { a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - a.key, err = getFrostFSKey(a) - if err != nil { - a.log.Fatal("failed to get frostfs credentials", zap.Error(err)) - } + a.pool, a.treePool, a.key = getPools(ctx, a.log, a.cfg) var owner user.ID user.IDFromKey(&owner, a.key.PrivateKey.PublicKey) a.owner = &owner - var prm pool.InitParameters - prm.SetKey(&a.key.PrivateKey) - prm.SetNodeDialTimeout(a.cfg.GetDuration(cfgConTimeout)) - prm.SetNodeStreamTimeout(a.cfg.GetDuration(cfgStreamTimeout)) - prm.SetHealthcheckTimeout(a.cfg.GetDuration(cfgReqTimeout)) - prm.SetClientRebalanceInterval(a.cfg.GetDuration(cfgRebalance)) - prm.SetErrorThreshold(a.cfg.GetUint32(cfgPoolErrorThreshold)) - - for i := 0; ; i++ { - address := a.cfg.GetString(cfgPeers + "." + strconv.Itoa(i) + ".address") - weight := a.cfg.GetFloat64(cfgPeers + "." + strconv.Itoa(i) + ".weight") - priority := a.cfg.GetInt(cfgPeers + "." + strconv.Itoa(i) + ".priority") - if address == "" { - break - } - if weight <= 0 { // unspecified or wrong - weight = 1 - } - if priority <= 0 { // unspecified or wrong - priority = 1 - } - prm.AddNode(pool.NewNodeParam(priority, address, weight)) - a.log.Info("add connection", zap.String("address", address), - zap.Float64("weight", weight), zap.Int("priority", priority)) - } - - a.pool, err = pool.NewPool(prm) - if err != nil { - a.log.Fatal("failed to create connection pool", zap.Error(err)) - } - - err = a.pool.Dial(ctx) - if err != nil { - a.log.Fatal("failed to dial pool", zap.Error(err)) - } - a.initAppSettings() a.initResolver() a.initMetrics() @@ -283,11 +239,11 @@ func remove(list []string, element string) []string { return list } -func getFrostFSKey(a *app) (*keys.PrivateKey, error) { - walletPath := a.cfg.GetString(cfgWalletPath) +func getFrostFSKey(cfg *viper.Viper, log *zap.Logger) (*keys.PrivateKey, error) { + walletPath := cfg.GetString(cfgWalletPath) if len(walletPath) == 0 { - a.log.Info("no wallet path specified, creating ephemeral key automatically for this run") + log.Info("no wallet path specified, creating ephemeral key automatically for this run") key, err := keys.NewPrivateKey() if err != nil { return nil, err @@ -300,12 +256,12 @@ func getFrostFSKey(a *app) (*keys.PrivateKey, error) { } var password *string - if a.cfg.IsSet(cfgWalletPassphrase) { - pwd := a.cfg.GetString(cfgWalletPassphrase) + if cfg.IsSet(cfgWalletPassphrase) { + pwd := cfg.GetString(cfgWalletPassphrase) password = &pwd } - address := a.cfg.GetString(cfgWalletAddress) + address := cfg.GetString(cfgWalletAddress) return getKeyFromWallet(w, address, password) } @@ -357,9 +313,8 @@ func (a *app) setHealthStatus() { } func (a *app) Serve() { - treeClient := a.initTree(a.ctx) uploadRoutes := uploader.New(a.AppParams(), a.settings.Uploader) - downloadRoutes := downloader.New(a.AppParams(), a.settings.Downloader, treeClient) + downloadRoutes := downloader.New(a.AppParams(), a.settings.Downloader, tree.NewTree(services.NewPoolWrapper(a.treePool))) // Configure router. a.configureRouter(uploadRoutes, downloadRoutes) @@ -599,19 +554,6 @@ func (a *app) serverIndex(address string) int { return -1 } -func (a *app) initTree(ctx context.Context) *tree.Tree { - treeServiceEndpoint := a.cfg.GetString(cfgTreeServiceEndpoint) - grpcDialOpt := grpc.WithTransportCredentials(insecure.NewCredentials()) - treeGRPCClient, err := services.NewTreeServiceClientGRPC(ctx, treeServiceEndpoint, a.key, grpcDialOpt) - if err != nil { - a.log.Fatal("failed to create tree service", zap.Error(err)) - } - treeService := tree.NewTree(treeGRPCClient) - a.log.Info("init tree service", zap.String("endpoint", treeServiceEndpoint)) - - return treeService -} - func (a *app) initTracing(ctx context.Context) { instanceID := "" if len(a.servers) > 0 { diff --git a/go.mod b/go.mod index 4532b1b..dd89afb 100644 --- a/go.mod +++ b/go.mod @@ -3,60 +3,60 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 - git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230608140155-9d40228cecbe github.com/fasthttp/router v1.4.1 - github.com/nspcc-dev/neo-go v0.101.0 - github.com/prometheus/client_golang v1.15.0 + github.com/nspcc-dev/neo-go v0.101.1 + github.com/prometheus/client_golang v1.15.1 github.com/prometheus/client_model v0.3.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 github.com/testcontainers/testcontainers-go v0.13.0 github.com/valyala/fasthttp v1.34.0 - go.opentelemetry.io/otel v1.14.0 - go.opentelemetry.io/otel/trace v1.14.0 + go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 - google.golang.org/grpc v1.53.0 - google.golang.org/protobuf v1.30.0 + google.golang.org/grpc v1.55.0 ) require ( git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb // indirect - git.frostfs.info/TrueCloudLab/hrw v1.2.0 // indirect + git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect + git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/hcsshim v0.9.2 // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/containerd v1.6.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/docker v20.10.14+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.8.0 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/klauspost/compress v1.16.4 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -81,31 +81,33 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect github.com/sirupsen/logrus v1.8.1 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/urfave/cli v1.22.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect - go.opentelemetry.io/otel/sdk v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/sdk v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.8.0 // indirect - golang.org/x/exp v0.0.0-20221227203929-1b447090c38c // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.7.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.9.0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 960fda0..6d0e55c 100644 --- a/go.sum +++ b/go.sum @@ -37,16 +37,18 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 h1:77lvdk0kMhnUgtnmqEcAPXPQaGlt24goMPu2+E5WRTk= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85/go.mod h1:sPyITTmQT662ZI38ud2aoE1SUCAr1mO5xV8P4nzLkKI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac h1:a6/Zc5BejflmguShwbllgJdEehnM9gshkLrLbKQHCU0= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd h1:HxacVl1Lc2RrfxAE13AGkp1tR/Mf4DDP6TgrgbLP5fQ= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd/go.mod h1:TaJJOF3Uhuq8aqv2CrfuY2yhxePUinW35Xd3wfXLV/I= -git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= -git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230608140155-9d40228cecbe h1:47lrWXcl36ayN7AJ9IW7sDDnTj//RUyHoIZOsjbYAYA= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230608140155-9d40228cecbe/go.mod h1:w+s3ozlbFfTDFHhjX0A3Iif3BRtnTkwiACxFZD+Q0cQ= +git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= +git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= @@ -119,8 +121,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 h1:npHgfD4Tl2WJS3AJaMUi5ynGDPUBfkg3U3fCzDyXZ+4= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -156,8 +158,8 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0Bsq github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= @@ -312,9 +314,8 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -392,8 +393,8 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -430,8 +431,8 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -524,8 +525,9 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -533,8 +535,9 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAszqCJMbfTISJ7oMftp8+UGV08= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= @@ -544,8 +547,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= -github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= +github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= @@ -684,8 +687,8 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= github.com/nspcc-dev/neo-go v0.99.4/go.mod h1:mKTolfRUfKjFso5HPvGSQtUZc70n0VKBMs16eGuC5gA= -github.com/nspcc-dev/neo-go v0.101.0 h1:JPT2DpZqVjho34TMR59dm6uxvCFttOp02Nm8qCjpfaU= -github.com/nspcc-dev/neo-go v0.101.0/go.mod h1:Q0uWKivGc2mYgdKFmTNP49LeXwMu4x6pUzHm3OIsN2I= +github.com/nspcc-dev/neo-go v0.101.1 h1:TVdcIpH/+bxQBTLRwWE3+Pw3j6j/JwguENbBSGAGid0= +github.com/nspcc-dev/neo-go v0.101.1/go.mod h1:J4tspxWw7jknX06F+VSMsKvIiNpYGfVTb2IxVC005YU= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb h1:GFxfkpXEYAbMIr69JpKOsQWeLOaGrd49HNAor8uDW+A= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= @@ -779,8 +782,8 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -819,8 +822,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -845,7 +848,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -886,8 +888,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -903,6 +905,8 @@ github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBT github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -957,20 +961,22 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 h1:/fXHZHGvro6MVqV34fJzDhi7sHGpX3Ej/Qjmfn003ho= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 h1:TKf2uAs2ueguzLaxOCBXNpHxfO/aC7PAdDsSH0IbeRQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 h1:sEL90JjOO/4yhquXl5zTAkLLsZ5+MycAgX99SDsxGc8= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0/go.mod h1:oCslUcizYdpKYyS9e8srZEqM6BB8fq41VJBjLAE6z1w= -go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= -go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 h1:+XWJd3jf75RXJq29mxbuXhCXFDG3S3R4vBUeSI2P7tE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0/go.mod h1:hqgzBPTf4yONMFgdZvL/bK42R/iinTyVQtiWihs3SZc= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -985,8 +991,8 @@ go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -1012,8 +1018,8 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1024,8 +1030,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221227203929-1b447090c38c h1:Govq2W3bnHJimHT2ium65kXcI7ZzTniZHcFATnLJM0Q= -golang.org/x/exp v0.0.0-20221227203929-1b447090c38c/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1106,8 +1112,8 @@ golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1132,8 +1138,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1233,13 +1239,13 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1398,8 +1404,8 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1425,8 +1431,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/integration_test.go b/integration_test.go index 014f923..34506c4 100644 --- a/integration_test.go +++ b/integration_test.go @@ -363,7 +363,6 @@ func getDefaultConfig() *viper.Viper { v.SetDefault(cfgRPCEndpoint, "http://localhost:30333") v.SetDefault("server.0.address", testListenAddress) - v.SetDefault(cfgTreeServiceEndpoint, "localhost:8080") return v } diff --git a/internal/frostfs/services/pool_wrapper.go b/internal/frostfs/services/pool_wrapper.go new file mode 100644 index 0000000..039d575 --- /dev/null +++ b/internal/frostfs/services/pool_wrapper.go @@ -0,0 +1,115 @@ +package services + +import ( + "context" + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" + treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" + grpcService "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree/service" +) + +type GetNodeByPathResponseInfoWrapper struct { + response *grpcService.GetNodeByPathResponse_Info +} + +func (n GetNodeByPathResponseInfoWrapper) GetNodeID() uint64 { + return n.response.GetNodeId() +} + +func (n GetNodeByPathResponseInfoWrapper) GetParentID() uint64 { + return n.response.GetParentId() +} + +func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() uint64 { + return n.response.GetTimestamp() +} + +func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { + res := make([]tree.Meta, len(n.response.Meta)) + for i, value := range n.response.Meta { + res[i] = value + } + return res +} + +type GetSubTreeResponseBodyWrapper struct { + response *grpcService.GetSubTreeResponse_Body +} + +func (n GetSubTreeResponseBodyWrapper) GetNodeID() uint64 { + return n.response.GetNodeId() +} + +func (n GetSubTreeResponseBodyWrapper) GetParentID() uint64 { + return n.response.GetParentId() +} + +func (n GetSubTreeResponseBodyWrapper) GetTimestamp() uint64 { + return n.response.GetTimestamp() +} + +func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta { + res := make([]tree.Meta, len(n.response.Meta)) + for i, value := range n.response.Meta { + res[i] = value + } + return res +} + +type PoolWrapper struct { + p *treepool.Pool +} + +func NewPoolWrapper(p *treepool.Pool) *PoolWrapper { + return &PoolWrapper{p: p} +} + +func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]tree.NodeResponse, error) { + poolPrm := treepool.GetNodesParams{ + CID: prm.CnrID, + TreeID: prm.TreeID, + Path: prm.Path, + Meta: prm.Meta, + PathAttribute: tree.FileNameKey, + LatestOnly: prm.LatestOnly, + AllAttrs: prm.AllAttrs, + BearerToken: getBearer(ctx), + } + + nodes, err := w.p.GetNodes(ctx, poolPrm) + if err != nil { + return nil, handleError(err) + } + + res := make([]tree.NodeResponse, len(nodes)) + for i, info := range nodes { + res[i] = GetNodeByPathResponseInfoWrapper{info} + } + + return res, nil +} + +func getBearer(ctx context.Context) []byte { + token, err := tokens.LoadBearerToken(ctx) + if err != nil { + return nil + } + return token.Marshal() +} + +func handleError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, treepool.ErrNodeNotFound) { + return fmt.Errorf("%w: %s", tree.ErrNodeNotFound, err.Error()) + } + if errors.Is(err, treepool.ErrNodeAccessDenied) { + return fmt.Errorf("%w: %s", tree.ErrNodeAccessDenied, err.Error()) + } + + return err +} diff --git a/settings.go b/settings.go index 7a78c53..eec014c 100644 --- a/settings.go +++ b/settings.go @@ -1,6 +1,8 @@ package main import ( + "context" + "encoding/hex" "fmt" "os" "path" @@ -11,11 +13,16 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) const ( @@ -59,9 +66,6 @@ const ( cfgRebalance = "rebalance_timer" cfgPoolErrorThreshold = "pool_error_threshold" - // Grpc path to tree service. - cfgTreeServiceEndpoint = "tree.service" - // Logger. cfgLoggerLevel = "logger.level" @@ -395,3 +399,107 @@ func fetchServers(v *viper.Viper) []ServerInfo { return servers } + +func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) { + key, err := getFrostFSKey(cfg, logger) + if err != nil { + logger.Fatal("could not load FrostFS private key", zap.Error(err)) + } + + var prm pool.InitParameters + var prmTree treepool.InitParameters + + prm.SetKey(&key.PrivateKey) + prmTree.SetKey(key) + logger.Info("using credentials", zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) + + for _, peer := range fetchPeers(logger, cfg) { + prm.AddNode(peer) + prmTree.AddNode(peer) + } + + connTimeout := cfg.GetDuration(cfgConTimeout) + if connTimeout <= 0 { + connTimeout = defaultConnectTimeout + } + prm.SetNodeDialTimeout(connTimeout) + prmTree.SetNodeDialTimeout(connTimeout) + + streamTimeout := cfg.GetDuration(cfgStreamTimeout) + if streamTimeout <= 0 { + streamTimeout = defaultStreamTimeout + } + prm.SetNodeStreamTimeout(streamTimeout) + prmTree.SetNodeStreamTimeout(streamTimeout) + + healthCheckTimeout := cfg.GetDuration(cfgReqTimeout) + if healthCheckTimeout <= 0 { + healthCheckTimeout = defaultRequestTimeout + } + prm.SetHealthcheckTimeout(healthCheckTimeout) + prmTree.SetHealthcheckTimeout(healthCheckTimeout) + + rebalanceInterval := cfg.GetDuration(cfgRebalance) + if rebalanceInterval <= 0 { + rebalanceInterval = defaultRebalanceTimer + } + prm.SetClientRebalanceInterval(rebalanceInterval) + prmTree.SetClientRebalanceInterval(rebalanceInterval) + + errorThreshold := cfg.GetUint32(cfgPoolErrorThreshold) + if errorThreshold <= 0 { + errorThreshold = defaultPoolErrorThreshold + } + prm.SetErrorThreshold(errorThreshold) + prm.SetLogger(logger) + prmTree.SetLogger(logger) + + p, err := pool.NewPool(prm) + if err != nil { + logger.Fatal("failed to create connection pool", zap.Error(err)) + } + + if err = p.Dial(ctx); err != nil { + logger.Fatal("failed to dial connection pool", zap.Error(err)) + } + + prmTree.SetGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())) + treePool, err := treepool.NewPool(prmTree) + if err != nil { + logger.Fatal("failed to create tree pool", zap.Error(err)) + } + if err = treePool.Dial(ctx); err != nil { + logger.Fatal("failed to dial tree pool", zap.Error(err)) + } + + return p, treePool, key +} + +func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam { + var nodes []pool.NodeParam + for i := 0; ; i++ { + key := cfgPeers + "." + strconv.Itoa(i) + "." + address := v.GetString(key + "address") + weight := v.GetFloat64(key + "weight") + priority := v.GetInt(key + "priority") + + if address == "" { + break + } + if weight <= 0 { // unspecified or wrong + weight = 1 + } + if priority <= 0 { // unspecified or wrong + priority = 1 + } + + nodes = append(nodes, pool.NewNodeParam(priority, address, weight)) + + l.Info("added storage peer", + zap.Int("priority", priority), + zap.String("address", address), + zap.Float64("weight", weight)) + } + + return nodes +} diff --git a/utils/tracing.go b/utils/tracing.go index 75a4486..14c059a 100644 --- a/utils/tracing.go +++ b/utils/tracing.go @@ -3,7 +3,7 @@ package utils import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "github.com/valyala/fasthttp" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" From 202ef5cc54ef2d18fc9b1a3abd6c22521c4767cc Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 9 Jun 2023 09:39:25 +0300 Subject: [PATCH 401/548] [#59] Drop sync-tree Signed-off-by: Denis Kirillov --- .forgejo/workflows/tests.yml | 6 - .forgejo/workflows/vulncheck.yml | 3 - .gitignore | 1 - Makefile | 8 +- internal/frostfs/services/tree_client_grpc.go | 114 ------------------ .../services/tree_client_grpc_signature.go | 29 ----- syncTree.sh | 21 ---- 7 files changed, 1 insertion(+), 181 deletions(-) delete mode 100644 internal/frostfs/services/tree_client_grpc.go delete mode 100644 internal/frostfs/services/tree_client_grpc_signature.go delete mode 100755 syncTree.sh diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index f64b816..7a03020 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -7,9 +7,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Sync tree service - run: make sync-tree - - name: golangci-lint uses: https://github.com/golangci/golangci-lint-action@v2 with: @@ -29,9 +26,6 @@ jobs: uses: actions/setup-go@v3 with: go-version: '${{ matrix.go_versions }}' - - - name: Sync tree service - run: make sync-tree - name: Update Go modules run: make dep diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 34692c9..0c9e908 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -9,9 +9,6 @@ jobs: with: fetch-depth: 0 - - name: Sync tree service - run: make sync-tree - - name: Setup Go uses: actions/setup-go@v3 with: diff --git a/.gitignore b/.gitignore index c1ca465..c4a98d8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ bin temp /plugins/ /vendor/ -internal/frostfs/services/tree .test.env *~ diff --git a/Makefile b/Makefile index aeea3a8..a4db526 100755 --- a/Makefile +++ b/Makefile @@ -15,7 +15,6 @@ METRICS_DUMP_OUT ?= ./metrics-dump.json BINDIR = bin DIRS = $(BINDIR) BINS = $(BINDIR)/frostfs-http-gw -SYNCDIR = internal/frostfs/services/tree .PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean @@ -28,7 +27,7 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ # Make all binaries all: $(BINS) -$(BINS): sync-tree $(DIRS) dep +$(BINS): $(DIRS) dep @echo "⇒ Build $@" CGO_ENABLED=0 \ go build -v -trimpath \ @@ -39,10 +38,6 @@ $(DIRS): @echo "⇒ Ensure dir: $@" @mkdir -p $@ -# Synchronize tree service -sync-tree: - @./syncTree.sh - # Pull go dependencies dep: @printf "⇒ Download requirements: " @@ -136,7 +131,6 @@ version: clean: rm -rf vendor rm -rf $(BINDIR) - rm -rf $(SYNCDIR) # Package for Debian debpackage: diff --git a/internal/frostfs/services/tree_client_grpc.go b/internal/frostfs/services/tree_client_grpc.go deleted file mode 100644 index f25588a..0000000 --- a/internal/frostfs/services/tree_client_grpc.go +++ /dev/null @@ -1,114 +0,0 @@ -package services - -import ( - "context" - "fmt" - "strings" - - grpcService "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services/tree" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "google.golang.org/grpc" -) - -type GetNodeByPathResponseInfoWrapper struct { - response *grpcService.GetNodeByPathResponse_Info -} - -func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { - res := make([]tree.Meta, len(n.response.Meta)) - for i, value := range n.response.Meta { - res[i] = value - } - return res -} - -type GetSubTreeResponseBodyWrapper struct { - response *grpcService.GetSubTreeResponse_Body -} - -func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta { - res := make([]tree.Meta, len(n.response.Meta)) - for i, value := range n.response.Meta { - res[i] = value - } - return res -} - -type ServiceClientGRPC struct { - key *keys.PrivateKey - conn *grpc.ClientConn - service grpcService.TreeServiceClient -} - -func NewTreeServiceClientGRPC(ctx context.Context, addr string, key *keys.PrivateKey, grpcOpts ...grpc.DialOption) (*ServiceClientGRPC, error) { - conn, err := grpc.Dial(addr, grpcOpts...) - if err != nil { - return nil, fmt.Errorf("did not connect: %v", err) - } - - c := grpcService.NewTreeServiceClient(conn) - if _, err = c.Healthcheck(ctx, &grpcService.HealthcheckRequest{}); err != nil { - return nil, fmt.Errorf("healthcheck: %w", err) - } - - return &ServiceClientGRPC{ - key: key, - conn: conn, - service: c, - }, nil -} - -func (c *ServiceClientGRPC) GetNodes(ctx context.Context, p *tree.GetNodesParams) ([]tree.NodeResponse, error) { - request := &grpcService.GetNodeByPathRequest{ - Body: &grpcService.GetNodeByPathRequest_Body{ - ContainerId: p.CnrID[:], - TreeId: p.TreeID, - Path: p.Path, - Attributes: p.Meta, - PathAttribute: tree.FileNameKey, - LatestOnly: p.LatestOnly, - AllAttributes: p.AllAttrs, - BearerToken: getBearer(ctx), - }, - } - - if err := c.signRequest(request.Body, func(key, sign []byte) { - request.Signature = &grpcService.Signature{ - Key: key, - Sign: sign, - } - }); err != nil { - return nil, err - } - - resp, err := c.service.GetNodeByPath(ctx, request) - if err != nil { - return nil, handleError("failed to get node by path", err) - } - - res := make([]tree.NodeResponse, len(resp.GetBody().GetNodes())) - for i, info := range resp.GetBody().GetNodes() { - res[i] = GetNodeByPathResponseInfoWrapper{info} - } - - return res, nil -} - -func getBearer(ctx context.Context) []byte { - token, err := tokens.LoadBearerToken(ctx) - if err != nil { - return nil - } - return token.Marshal() -} - -func handleError(msg string, err error) error { - if strings.Contains(err.Error(), "not found") { - return fmt.Errorf("%w: %s", tree.ErrNodeNotFound, err.Error()) - } else if strings.Contains(err.Error(), "is denied by") { - return fmt.Errorf("%w: %s", tree.ErrNodeAccessDenied, err.Error()) - } - return fmt.Errorf("%s: %w", msg, err) -} diff --git a/internal/frostfs/services/tree_client_grpc_signature.go b/internal/frostfs/services/tree_client_grpc_signature.go deleted file mode 100644 index 9dd38f9..0000000 --- a/internal/frostfs/services/tree_client_grpc_signature.go +++ /dev/null @@ -1,29 +0,0 @@ -/*REMOVE THIS AFTER SIGNATURE WILL BE AVAILABLE IN TREE CLIENT FROM FROSTFS NODE*/ -package services - -import ( - crypto "git.frostfs.info/TrueCloudLab/frostfs-crypto" - "google.golang.org/protobuf/proto" -) - -func (c *ServiceClientGRPC) signData(buf []byte, f func(key, sign []byte)) error { - // crypto package should not be used outside of API libraries (see neofs-node#491). - // For now tree service does not include into SDK Client nor SDK Pool, so there is no choice. - // When SDK library adopts Tree service client, this should be dropped. - sign, err := crypto.Sign(&c.key.PrivateKey, buf) - if err != nil { - return err - } - - f(c.key.PublicKey().Bytes(), sign) - return nil -} - -func (c *ServiceClientGRPC) signRequest(requestBody proto.Message, f func(key, sign []byte)) error { - buf, err := proto.Marshal(requestBody) - if err != nil { - return err - } - - return c.signData(buf, f) -} diff --git a/syncTree.sh b/syncTree.sh deleted file mode 100755 index 98d16a0..0000000 --- a/syncTree.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -mkdir -p internal/frostfs/services/tree 2>/dev/null - -REVISION="f07d4158f50ed5c7f44cc0bc224c3d03edf27f3b" - -echo "tree service revision ${REVISION}" - -# regexp below find all link to source code files which end with ".pb.go" and retrieve the file names -# we use `[^.]*` as non greedy workaround for `.*` -FILES=$(curl -s https://git.frostfs.info/TrueCloudLab/frostfs-node/src/commit/${REVISION}/pkg/services/tree | sed -n "s,.*\"/TrueCloudLab/frostfs-node/src/commit/${REVISION}/pkg/services/tree/\([^.]*\.pb\.go\)\".*,\1,p") - -for file in $FILES; do - if [[ $file == *"frostfs"* ]]; then - echo "skip '$file'" - continue - else - echo "sync '$file' in tree service" - fi - curl -s "https://git.frostfs.info/TrueCloudLab/frostfs-node/raw/commit/${REVISION}/pkg/services/tree/${file}" -o "./internal/frostfs/services/tree/${file}" -done \ No newline at end of file From 2ccb43bc8cfcb38fc968c83679e7e2aef5bf29cc Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 9 Jun 2023 09:39:40 +0300 Subject: [PATCH 402/548] [#59] Update docs Signed-off-by: Denis Kirillov --- CHANGELOG.md | 3 +++ config/config.env | 5 +---- config/config.yaml | 4 ---- docs/gate-configuration.md | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11d43e8..ff51ce8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ This document outlines major changes between releases. - Separate integration tests with build tags (#24) - Changed values for `frostfs_http_gw_state_health` metric (#32) +### Removed +- Drop `tree.service` param (now endpoints from `peers` section are used) (#59) + ### Updating from neofs-http-gw v0.26.0 To set system attributes use updated headers diff --git a/config/config.env b/config/config.env index 4dc9bb2..debdca2 100644 --- a/config/config.env +++ b/config/config.env @@ -93,9 +93,6 @@ HTTP_GW_POOL_ERROR_THRESHOLD=100 # Enable zip compression to download files by common prefix. HTTP_GW_ZIP_COMPRESSION=false -# Endpoint of the tree service. Must be provided. Can be one of the node address (from the `peers` section). -HTTP_GW_TREE_SERVICE=grpc://s01.frostfs.devenv:8080 - HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENDPOINT="localhost:4317" -HTTP_GW_TRACING_EXPORTER="otlp_grpc" \ No newline at end of file +HTTP_GW_TRACING_EXPORTER="otlp_grpc" diff --git a/config/config.yaml b/config/config.yaml index a71c69d..510cb43 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -93,10 +93,6 @@ resolve_order: upload_header: use_default_timestamp: false # Create timestamp for object if it isn't provided by header. -# Endpoint of the tree service. Must be provided. Can be one of the node address (from the `peers` section). -tree: - service: 127.0.0.1:8080 - connect_timeout: 5s # Timeout to dial node. stream_timeout: 10s # Timeout for individual operations in streaming RPC. request_timeout: 5s # Timeout to check node health during rebalance. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 0d0504f..5b4edae 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -52,7 +52,7 @@ $ cat http.log | `zip` | [ZIP configuration](#zip-section) | | `pprof` | [Pprof configuration](#pprof-section) | | `prometheus` | [Prometheus configuration](#prometheus-section) | -| `tracing` | [Tracing configuration](#tracing-section) | +| `tracing` | [Tracing configuration](#tracing-section) | # General section From 6f64557a4bf57d69aafe010cb8f0d05a8b572c07 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 14 Jun 2023 11:27:33 +0300 Subject: [PATCH 403/548] [#60] Use gRPC interceptor from observability package Previous SDK implementation had implicit gRPC interceptor for tracing. Now pool constructors allow any dial options, so gateway should manually pass tracing interceptors from observability package. Signed-off-by: Alex Vanin --- settings.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/settings.go b/settings.go index eec014c..3b2cba9 100644 --- a/settings.go +++ b/settings.go @@ -13,6 +13,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -454,6 +455,19 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool. prm.SetLogger(logger) prmTree.SetLogger(logger) + var apiGRPCDialOpts []grpc.DialOption + var treeGRPCDialOpts = []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + if cfg.GetBool(cfgTracingEnabled) { + interceptors := []grpc.DialOption{ + grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), + grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), + } + treeGRPCDialOpts = append(treeGRPCDialOpts, interceptors...) + apiGRPCDialOpts = append(apiGRPCDialOpts, interceptors...) + } + prm.SetGRPCDialOptions(apiGRPCDialOpts...) + prmTree.SetGRPCDialOptions(treeGRPCDialOpts...) + p, err := pool.NewPool(prm) if err != nil { logger.Fatal("failed to create connection pool", zap.Error(err)) @@ -463,7 +477,6 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool. logger.Fatal("failed to dial connection pool", zap.Error(err)) } - prmTree.SetGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())) treePool, err := treepool.NewPool(prmTree) if err != nil { logger.Fatal("failed to create tree pool", zap.Error(err)) From d9122e20939ace7833f572fcc19c57ec8a777d5e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 7 Jul 2023 16:40:13 +0300 Subject: [PATCH 404/548] [#62] Update sdk to support grpc schemes in tree pool Signed-off-by: Denis Kirillov --- go.mod | 4 ++-- go.sum | 8 ++++---- settings.go | 3 +-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index dd89afb..e51b7ed 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230608140155-9d40228cecbe + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230707115716-fe35373d8f1b github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.1 github.com/prometheus/client_golang v1.15.1 diff --git a/go.sum b/go.sum index 6d0e55c..0b13573 100644 --- a/go.sum +++ b/go.sum @@ -37,16 +37,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac h1:a6/Zc5BejflmguShwbllgJdEehnM9gshkLrLbKQHCU0= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe h1:SB102RiEg+4h9qcwyG97zHBtwduMRbedbtkwRDVSps8= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230608140155-9d40228cecbe h1:47lrWXcl36ayN7AJ9IW7sDDnTj//RUyHoIZOsjbYAYA= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230608140155-9d40228cecbe/go.mod h1:w+s3ozlbFfTDFHhjX0A3Iif3BRtnTkwiACxFZD+Q0cQ= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230707115716-fe35373d8f1b h1:nFcOeS2lMmuFdouEqA9SrmzvU/gt46VSjiBj2zZpMQ8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230707115716-fe35373d8f1b/go.mod h1:r5Fir/4jCVXzdfOyCUbikSDB99nVqnHNq7mzVcidnlA= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/settings.go b/settings.go index 3b2cba9..c402335 100644 --- a/settings.go +++ b/settings.go @@ -23,7 +23,6 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) const ( @@ -456,7 +455,7 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool. prmTree.SetLogger(logger) var apiGRPCDialOpts []grpc.DialOption - var treeGRPCDialOpts = []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + var treeGRPCDialOpts []grpc.DialOption if cfg.GetBool(cfgTracingEnabled) { interceptors := []grpc.DialOption{ grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), From 6fac6341c21d557677154b6953ddc5ff77542de1 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 12 Jul 2023 11:30:50 +0300 Subject: [PATCH 405/548] Release v0.27.0 Signed-off-by: Alex Vanin --- CHANGELOG.md | 36 +++++++++++++++++++++++++++++------- VERSION | 2 +- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff51ce8..48463c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,25 +4,46 @@ This document outlines major changes between releases. ## [Unreleased] +### Fixed +- `grpc` schemas in tree configuration (#62) + ### Added -- Multiple configs support (TrueCloudLab#12) - Support dump metrics descriptions (#29) -- Support impersonate bearer token (#40) +- Support impersonate bearer token (#40, #45) +- Tracing support (#20, #44, #60) +- Object name resolving with tree service (#30) ### Changed - Update prometheus to v1.15.0 (#35) -- Update go version to 1.18 (TrueCloudLab#9) - Update go version to 1.19 (#50) +- Finish rebranding (#2) + +### Removed +- Drop `tree.service` param (now endpoints from `peers` section are used) (#59) + +## [0.27.0] - Karpinsky - 2023-07-12 + +This is a first FrostFS HTTP Gateway release named after +[Karpinsky glacier](https://en.wikipedia.org/wiki/Karpinsky_Glacier). + +### Fixed +- Require only one healthy storage server to start (#7) +- Enable gate metrics (#38) +- `Too many pings` error (#61) + +### Added +- Multiple configs support (#12) + +### Changed +- Repository rebranding (#1) - Update neo-go to v0.101.0 (#8) - Update viper to v1.15.0 (#8) +- Update go version to 1.18 (#9) - Errors have become more detailed (#18) - Update system attribute names (#22) - Separate integration tests with build tags (#24) - Changed values for `frostfs_http_gw_state_health` metric (#32) -### Removed -- Drop `tree.service` param (now endpoints from `peers` section are used) (#59) - ### Updating from neofs-http-gw v0.26.0 To set system attributes use updated headers @@ -38,4 +59,5 @@ To set system attributes use updated headers This project is a fork of [NeoFS HTTP Gateway](https://github.com/nspcc-dev/neofs-http-gw) from version v0.26.0. To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs-http-gw/blob/master/CHANGELOG.md. -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/72734ab4...master +[0.27.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/72734ab4...v0.27.0 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.27.0...master diff --git a/VERSION b/VERSION index eaf8bae..0a8bf80 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.26.0 +v0.27.0 From 0882d344a2b9dce327b9cd0791031e2f613db0c7 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 11 Jul 2023 13:25:55 +0300 Subject: [PATCH 406/548] [#63] Use forked actions during workflows Signed-off-by: Alex Vanin --- .forgejo/workflows/dco.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index b1ccd0e..6cbe4dd 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -15,6 +15,6 @@ jobs: go-version: '1.20' - name: Run commit format checker - uses: https://git.alexvan.in/alexvanin/dco-go@v1 + uses: https://git.frostfs.info/TrueCloudLab/dco-go@v1 with: from: adb95642d From 97ac638dff829966d285c361198c85e4e5d74789 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 2 Aug 2023 17:11:44 +0300 Subject: [PATCH 407/548] [#67] Fix GetSubTree failures with updated SDK Signed-off-by: Alex Vanin --- go.mod | 8 ++++---- go.sum | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index e51b7ed..632997f 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230707115716-fe35373d8f1b + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6 github.com/fasthttp/router v1.4.1 - github.com/nspcc-dev/neo-go v0.101.1 + github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc github.com/prometheus/client_golang v1.15.1 github.com/prometheus/client_model v0.3.0 github.com/spf13/pflag v1.0.5 @@ -68,7 +68,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect diff --git a/go.sum b/go.sum index 0b13573..f4c8c56 100644 --- a/go.sum +++ b/go.sum @@ -37,16 +37,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe h1:SB102RiEg+4h9qcwyG97zHBtwduMRbedbtkwRDVSps8= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 h1:v6JqBD/VzZx3QSxbaXnUwnnJ1KEYheU4LzLGr3IhsAE= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230707115716-fe35373d8f1b h1:nFcOeS2lMmuFdouEqA9SrmzvU/gt46VSjiBj2zZpMQ8= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230707115716-fe35373d8f1b/go.mod h1:r5Fir/4jCVXzdfOyCUbikSDB99nVqnHNq7mzVcidnlA= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6 h1:u6lzNotV6MEMNEG/XeS7g+FjPrrf+j4gnOHtvun2KJc= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6/go.mod h1:LI2GOj0pEx0jYTjB3QHja2PNhQFYL2pCm71RAFwDv0M= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= @@ -369,7 +369,7 @@ github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:r github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -687,11 +687,11 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= github.com/nspcc-dev/neo-go v0.99.4/go.mod h1:mKTolfRUfKjFso5HPvGSQtUZc70n0VKBMs16eGuC5gA= -github.com/nspcc-dev/neo-go v0.101.1 h1:TVdcIpH/+bxQBTLRwWE3+Pw3j6j/JwguENbBSGAGid0= -github.com/nspcc-dev/neo-go v0.101.1/go.mod h1:J4tspxWw7jknX06F+VSMsKvIiNpYGfVTb2IxVC005YU= +github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc h1:fySIWvUQsitK5e5qYIHnTDCXuPpwzz89SEUEIyY11sg= +github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc/go.mod h1:s9QhjMC784MWqTURovMbyYduIJc86mnCruxcMiAebpc= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb h1:GFxfkpXEYAbMIr69JpKOsQWeLOaGrd49HNAor8uDW+A= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce h1:vLGuUNDkmQrWMa4rr4vTd1u8ULqejWxVmNz1L7ocTEI= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= From cc69601b32a09290dae3ccdeab4eb39515251172 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 7 Aug 2023 12:08:34 +0300 Subject: [PATCH 408/548] [#66] Use gate key to form object owner This is required because node check session token owner https://git.frostfs.info/TrueCloudLab/frostfs-node/pulls/528 For client cut https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/114 such owner will be gate owner Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + uploader/upload.go | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48463c1..d323c89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ This document outlines major changes between releases. - Update prometheus to v1.15.0 (#35) - Update go version to 1.19 (#50) - Finish rebranding (#2) +- Use gate key to form object owner (#66) ### Removed - Drop `tree.service` param (now endpoints from `peers` section are used) (#59) diff --git a/uploader/upload.go b/uploader/upload.go index 43996ea..57e426d 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -145,17 +145,17 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) attributes = append(attributes, *timestamp) } - id, bt := u.fetchOwnerAndBearerToken(ctx) obj := object.New() obj.SetContainerID(*idCnr) - obj.SetOwnerID(id) + obj.SetOwnerID(u.ownerID) obj.SetAttributes(attributes...) var prm pool.PrmObjectPut prm.SetHeader(*obj) prm.SetPayload(file) + bt := u.fetchBearerToken(ctx) if bt != nil { prm.UseBearer(*bt) } @@ -200,12 +200,11 @@ func (u *Uploader) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) { response.Error(r, msg, statusCode) } -func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*user.ID, *bearer.Token) { +func (u *Uploader) fetchBearerToken(ctx context.Context) *bearer.Token { if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil { - issuer := bearer.ResolveIssuer(*tkn) - return &issuer, tkn + return tkn } - return u.ownerID, nil + return nil } type putResponse struct { From fa28f1ff824feb6635f44f49acf99d84bb5f6be6 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Sun, 27 Aug 2023 18:09:02 +0300 Subject: [PATCH 409/548] [#36] Move log messages to constants Signed-off-by: Roman Loginov --- app.go | 59 ++++++++++++++++---------------- downloader/download.go | 39 ++++++++++----------- downloader/head.go | 3 +- internal/logs/logs.go | 69 ++++++++++++++++++++++++++++++++++++++ metrics/service.go | 11 +++--- settings.go | 15 +++++---- uploader/filter.go | 3 +- uploader/multipart.go | 5 +-- uploader/multipart_test.go | 5 +-- uploader/upload.go | 17 +++++----- 10 files changed, 152 insertions(+), 74 deletions(-) create mode 100644 internal/logs/logs.go diff --git a/app.go b/app.go index 88f9a07..864f39c 100644 --- a/app.go +++ b/app.go @@ -12,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/downloader" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" @@ -148,7 +149,7 @@ func (a *app) initResolver() { var err error a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) if err != nil { - a.log.Fatal("failed to create resolver", zap.Error(err)) + a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err)) } } @@ -161,11 +162,11 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) { order := a.cfg.GetStringSlice(cfgResolveOrder) if resolveCfg.RPCAddress == "" { order = remove(order, resolver.NNSResolver) - a.log.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint)) + a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided) } if len(order) == 0 { - a.log.Info("container resolver will be disabled because of resolvers 'resolver_order' is empty") + a.log.Info(logs.ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty) } return order, resolveCfg @@ -179,7 +180,7 @@ func (a *app) initMetrics() { func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics { if !enabled { - logger.Warn("metrics are disabled") + logger.Warn(logs.MetricsAreDisabled) } return &gateMetrics{ logger: logger, @@ -190,7 +191,7 @@ func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled b func (m *gateMetrics) SetEnabled(enabled bool) { if !enabled { - m.logger.Warn("metrics are disabled") + m.logger.Warn(logs.MetricsAreDisabled) } m.mu.Lock() @@ -243,7 +244,7 @@ func getFrostFSKey(cfg *viper.Viper, log *zap.Logger) (*keys.PrivateKey, error) walletPath := cfg.GetString(cfgWalletPath) if len(walletPath) == 0 { - log.Info("no wallet path specified, creating ephemeral key automatically for this run") + log.Info(logs.NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun) key, err := keys.NewPrivateKey() if err != nil { return nil, err @@ -300,7 +301,7 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*keys } func (a *app) Wait() { - a.log.Info("starting application", zap.String("app_name", "frostfs-http-gw"), zap.String("version", Version)) + a.log.Info(logs.StartingApplication, zap.String("app_name", "frostfs-http-gw"), zap.String("version", Version)) a.metrics.SetVersion(Version) a.setHealthStatus() @@ -324,9 +325,9 @@ func (a *app) Serve() { for i := range a.servers { go func(i int) { - a.log.Info("starting server", zap.String("address", a.servers[i].Address())) + a.log.Info(logs.StartingServer, zap.String("address", a.servers[i].Address())) if err := a.webServer.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed { - a.log.Fatal("listen and serve", zap.Error(err)) + a.log.Fatal(logs.ListenAndServe, zap.Error(err)) } }(i) } @@ -344,7 +345,7 @@ LOOP: } } - a.log.Info("shutting down web server", zap.Error(a.webServer.Shutdown())) + a.log.Info(logs.ShuttingDownWebServer, zap.Error(a.webServer.Shutdown())) a.metrics.Shutdown() a.stopServices() @@ -359,33 +360,33 @@ func (a *app) shutdownTracing() { defer cancel() if err := tracing.Shutdown(shdnCtx); err != nil { - a.log.Warn("failed to shutdown tracing", zap.Error(err)) + a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err)) } } func (a *app) configReload(ctx context.Context) { - a.log.Info("SIGHUP config reload started") + a.log.Info(logs.SIGHUPConfigReloadStarted) if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) { - a.log.Warn("failed to reload config because it's missed") + a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed) return } if err := readInConfig(a.cfg); err != nil { - a.log.Warn("failed to reload config", zap.Error(err)) + a.log.Warn(logs.FailedToReloadConfig, zap.Error(err)) return } if lvl, err := getLogLevel(a.cfg); err != nil { - a.log.Warn("log level won't be updated", zap.Error(err)) + a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err)) } else { a.logLevel.SetLevel(lvl) } if err := a.resolver.UpdateResolvers(a.getResolverConfig()); err != nil { - a.log.Warn("failed to update resolvers", zap.Error(err)) + a.log.Warn(logs.FailedToUpdateResolvers, zap.Error(err)) } if err := a.updateServers(); err != nil { - a.log.Warn("failed to reload server parameters", zap.Error(err)) + a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err)) } a.stopServices() @@ -397,7 +398,7 @@ func (a *app) configReload(ctx context.Context) { a.initTracing(ctx) a.setHealthStatus() - a.log.Info("SIGHUP config reload completed") + a.log.Info(logs.SIGHUPConfigReloadCompleted) } func (a *app) updateSettings() { @@ -436,22 +437,22 @@ func (a *app) configureRouter(uploadRoutes *uploader.Uploader, downloadRoutes *d response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } r.POST("/upload/{cid}", a.logger(a.tokenizer(a.tracer(uploadRoutes.Upload)))) - a.log.Info("added path /upload/{cid}") + a.log.Info(logs.AddedPathUploadCid) r.GET("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.DownloadByAddressOrBucketName)))) r.HEAD("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.HeadByAddressOrBucketName)))) - a.log.Info("added path /get/{cid}/{oid}") + a.log.Info(logs.AddedPathGetCidOid) r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.DownloadByAttribute)))) r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.HeadByAttribute)))) - a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") + a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) r.GET("/zip/{cid}/{prefix:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.DownloadZipped)))) - a.log.Info("added path /zip/{cid}/{prefix}") + a.log.Info(logs.AddedPathZipCidPrefix) a.webServer.Handler = r.Handler } func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { - a.log.Info("request", zap.String("remote", req.RemoteAddr().String()), + a.log.Info(logs.Request, zap.String("remote", req.RemoteAddr().String()), zap.ByteString("method", req.Method()), zap.ByteString("path", req.Path()), zap.ByteString("query", req.QueryArgs().QueryString()), @@ -464,7 +465,7 @@ func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { appCtx, err := tokens.StoreBearerTokenAppCtx(a.ctx, req) if err != nil { - a.log.Error("could not fetch and store bearer token", zap.Error(err)) + a.log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err)) response.Error(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) } utils.SetContextToRequest(appCtx, req) @@ -507,16 +508,16 @@ func (a *app) initServers(ctx context.Context) { } srv, err := newServer(ctx, serverInfo) if err != nil { - a.log.Warn("failed to add server", append(fields, zap.Error(err))...) + a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...) continue } a.servers = append(a.servers, srv) - a.log.Info("add server", fields...) + a.log.Info(logs.AddServer, fields...) } if len(a.servers) == 0 { - a.log.Fatal("no healthy servers") + a.log.Fatal(logs.NoHealthyServers) } } @@ -569,9 +570,9 @@ func (a *app) initTracing(ctx context.Context) { } updated, err := tracing.Setup(ctx, cfg) if err != nil { - a.log.Warn("failed to initialize tracing", zap.Error(err)) + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) } if updated { - a.log.Info("tracing config updated") + a.log.Info(logs.TracingConfigUpdated) } } diff --git a/downloader/download.go b/downloader/download.go index 2a08c82..cd7f72f 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" @@ -134,7 +135,7 @@ func receiveFile(ctx context.Context, req request, clnt *pool.Pool, objectAddres case object.AttributeTimestamp: value, err := strconv.ParseInt(val, 10, 64) if err != nil { - req.log.Info("couldn't parse creation date", + req.log.Info(logs.CouldntParseCreationDate, zap.String("key", key), zap.String("val", val), zap.Error(err)) @@ -157,7 +158,7 @@ func receiveFile(ctx context.Context, req request, clnt *pool.Pool, objectAddres return rObj.Payload, nil }) if err != nil && err != io.EOF { - req.log.Error("could not detect Content-Type from payload", zap.Error(err)) + req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) response.Error(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -195,7 +196,7 @@ func (r *request) handleFrostFSErr(err error, start time.Time) { statusCode, msg, additionalFields := response.FormErrorResponse("could not receive object", err) logFields = append(logFields, additionalFields...) - r.log.Error("could not receive object", logFields...) + r.log.Error(logs.CouldNotReceiveObject, logFields...) response.Error(r.RequestCtx, msg, statusCode) } @@ -264,14 +265,14 @@ func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(context.Context, r cnrID, err := utils.GetContainerID(ctx, idCnr, d.containerResolver) if err != nil { - log.Error("wrong container id", zap.Error(err)) + log.Error(logs.WrongContainerID, zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } objID := new(oid.ID) if err = objID.DecodeString(idObj); err != nil { - log.Error("wrong object id", zap.Error(err)) + log.Error(logs.WrongObjectID, zap.Error(err)) response.Error(c, "wrong object id", fasthttp.StatusBadRequest) return } @@ -296,19 +297,19 @@ func (d *Downloader) byBucketname(req *fasthttp.RequestCtx, f func(context.Conte cnrID, err := utils.GetContainerID(ctx, bucketname, d.containerResolver) if err != nil { - log.Error("wrong container id", zap.Error(err)) + log.Error(logs.WrongContainerID, zap.Error(err)) response.Error(req, "wrong container id", fasthttp.StatusBadRequest) return } foundOid, err := d.tree.GetLatestVersion(ctx, cnrID, key) if err != nil { - log.Error("object wasn't found", zap.Error(err)) + log.Error(logs.ObjectWasntFound, zap.Error(err)) response.Error(req, "object wasn't found", fasthttp.StatusNotFound) return } if foundOid.DeleteMarker { - log.Error("object was deleted") + log.Error(logs.ObjectWasDeleted) response.Error(req, "object deleted", fasthttp.StatusNotFound) return } @@ -338,14 +339,14 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, containerID, err := utils.GetContainerID(ctx, scid, d.containerResolver) if err != nil { - log.Error("wrong container id", zap.Error(err)) + log.Error(logs.WrongContainerID, zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } res, err := d.search(ctx, containerID, key, val, object.MatchStringEqual) if err != nil { - log.Error("could not search for objects", zap.Error(err)) + log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -357,12 +358,12 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, n, err := res.Read(buf) if n == 0 { if errors.Is(err, io.EOF) { - log.Error("object not found", zap.Error(err)) + log.Error(logs.ObjectNotFound, zap.Error(err)) response.Error(c, "object not found", fasthttp.StatusNotFound) return } - log.Error("read object list failed", zap.Error(err)) + log.Error(logs.ReadObjectListFailed, zap.Error(err)) response.Error(c, "read object list failed: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -424,7 +425,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { containerID, err := utils.GetContainerID(ctx, scid, d.containerResolver) if err != nil { - log.Error("wrong container id", zap.Error(err)) + log.Error(logs.WrongContainerID, zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } @@ -433,7 +434,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { // otherwise we get this error only in object iteration step // and client get 200 OK. if _, err = d.getContainer(ctx, *containerID); err != nil { - log.Error("could not check container existence", zap.Error(err)) + log.Error(logs.CouldNotCheckContainerExistence, zap.Error(err)) if client.IsErrContainerNotFound(err) { response.Error(c, "Not Found", fasthttp.StatusNotFound) return @@ -444,7 +445,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { resSearch, err := d.search(ctx, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { - log.Error("could not search for objects", zap.Error(err)) + log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -476,19 +477,19 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { addr.SetObject(id) if err = d.zipObject(ctx, zipWriter, addr, btoken, bufZip); err != nil { - log.Error("failed to add object to archive", zap.String("oid", id.EncodeToString()), zap.Error(err)) + log.Error(logs.FailedToAddObjectToArchive, zap.String("oid", id.EncodeToString()), zap.Error(err)) } return false }) if errIter != nil { - log.Error("iterating over selected objects failed", zap.Error(errIter)) + log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) } else if !called { - log.Error("objects not found") + log.Error(logs.ObjectsNotFound) } if err = zipWriter.Close(); err != nil { - log.Error("close zip writer", zap.Error(err)) + log.Error(logs.CloseZipWriter, zap.Error(err)) } }) } diff --git a/downloader/head.go b/downloader/head.go index 3e5a92a..76dfd93 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -7,6 +7,7 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -57,7 +58,7 @@ func headObject(ctx context.Context, req request, clnt *pool.Pool, objectAddress case object.AttributeTimestamp: value, err := strconv.ParseInt(val, 10, 64) if err != nil { - req.log.Info("couldn't parse creation date", + req.log.Info(logs.CouldntParseCreationDate, zap.String("key", key), zap.String("val", val), zap.Error(err)) diff --git a/internal/logs/logs.go b/internal/logs/logs.go new file mode 100644 index 0000000..d47a2c3 --- /dev/null +++ b/internal/logs/logs.go @@ -0,0 +1,69 @@ +package logs + +const ( + CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/* + CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go + CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go + WrongContainerID = "wrong container id" // Error in ../../downloader/download.go and uploader/upload.go + WrongObjectID = "wrong object id" // Error in ../../downloader/download.go + ObjectWasntFound = "object wasn't found" // Error in ../../downloader/download.go + ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go + CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go + ObjectNotFound = "object not found" // Error in ../../downloader/download.go + ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go + CouldNotCheckContainerExistence = "could not check container existence" // Error in ../../downloader/download.go + FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go + IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go + ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go + CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go + ServiceIsRunning = "service is running" // Info in ../../metrics/service.go + ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go + ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go + ShuttingDownService = "shutting down service" // Info in ../../metrics/service.go + CantShutDownService = "can't shut down service" // Panic in ../../metrics/service.go + IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go + IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go + CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go + CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go + CouldNotProcessHeaders = "could not process headers" // Error in ../../uploader/upload.go + CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go + CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go + CouldNotEncodeResponse = "could not encode response" // Error in ../../uploader/upload.go + CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go + AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go + FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go + ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go + MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go + NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" // Info in ../../app.go + StartingApplication = "starting application" // Info in ../../app.go + StartingServer = "starting server" // Info in ../../app.go + ListenAndServe = "listen and serve" // Fatal in ../../app.go + ShuttingDownWebServer = "shutting down web server" // Info in ../../app.go + FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../app.go + SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../app.go + FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../app.go + FailedToReloadConfig = "failed to reload config" // Warn in ../../app.go + LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../app.go + FailedToUpdateResolvers = "failed to update resolvers" // Warn in ../../app.go + FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../app.go + SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../app.go + AddedPathUploadCid = "added path /upload/{cid}" // Info in ../../app.go + AddedPathGetCidOid = "added path /get/{cid}/{oid}" // Info in ../../app.go + AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" // Info in ../../app.go + AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" // Info in ../../app.go + Request = "request" // Info in ../../app.go + CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" // Error in ../../app.go + FailedToAddServer = "failed to add server" // Warn in ../../app.go + AddServer = "add server" // Info in ../../app.go + NoHealthyServers = "no healthy servers" // Fatal in ../../app.go + FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../app.go + TracingConfigUpdated = "tracing config updated" // Info in ../../app.go + ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" // Warn in ../../app.go + CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../settings.go + UsingCredentials = "using credentials" // Info in ../../settings.go + FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../settings.go + FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../settings.go + FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../settings.go + FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../settings.go + AddedStoragePeer = "added storage peer" // Info in ../../settings.go +) diff --git a/metrics/service.go b/metrics/service.go index 7cb3ca2..c025f06 100644 --- a/metrics/service.go +++ b/metrics/service.go @@ -4,6 +4,7 @@ import ( "context" "net/http" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "go.uber.org/zap" ) @@ -24,21 +25,21 @@ type Config struct { // Start runs http service with the exposed endpoint on the configured port. func (ms *Service) Start() { if ms.enabled { - ms.log.Info("service is running", zap.String("endpoint", ms.Addr)) + ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr)) err := ms.ListenAndServe() if err != nil && err != http.ErrServerClosed { - ms.log.Warn("service couldn't start on configured port") + ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort) } } else { - ms.log.Info("service hasn't started since it's disabled") + ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled) } } // ShutDown stops the service. func (ms *Service) ShutDown(ctx context.Context) { - ms.log.Info("shutting down service", zap.String("endpoint", ms.Addr)) + ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr)) err := ms.Shutdown(ctx) if err != nil { - ms.log.Panic("can't shut down service") + ms.log.Panic(logs.CantShutDownService) } } diff --git a/settings.go b/settings.go index c402335..ed13da2 100644 --- a/settings.go +++ b/settings.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" @@ -403,7 +404,7 @@ func fetchServers(v *viper.Viper) []ServerInfo { func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) { key, err := getFrostFSKey(cfg, logger) if err != nil { - logger.Fatal("could not load FrostFS private key", zap.Error(err)) + logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) } var prm pool.InitParameters @@ -411,7 +412,7 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool. prm.SetKey(&key.PrivateKey) prmTree.SetKey(key) - logger.Info("using credentials", zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) + logger.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) for _, peer := range fetchPeers(logger, cfg) { prm.AddNode(peer) @@ -469,19 +470,19 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool. p, err := pool.NewPool(prm) if err != nil { - logger.Fatal("failed to create connection pool", zap.Error(err)) + logger.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) } if err = p.Dial(ctx); err != nil { - logger.Fatal("failed to dial connection pool", zap.Error(err)) + logger.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) } treePool, err := treepool.NewPool(prmTree) if err != nil { - logger.Fatal("failed to create tree pool", zap.Error(err)) + logger.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) } if err = treePool.Dial(ctx); err != nil { - logger.Fatal("failed to dial tree pool", zap.Error(err)) + logger.Fatal(logs.FailedToDialTreePool, zap.Error(err)) } return p, treePool, key @@ -507,7 +508,7 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam { nodes = append(nodes, pool.NewNodeParam(priority, address, weight)) - l.Info("added storage peer", + l.Info(logs.AddedStoragePeer, zap.Int("priority", priority), zap.String("address", address), zap.Float64("weight", weight)) diff --git a/uploader/filter.go b/uploader/filter.go index 35de625..70d6eef 100644 --- a/uploader/filter.go +++ b/uploader/filter.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -47,7 +48,7 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st result[k] = v - l.Debug("add attribute to result object", + l.Debug(logs.AddAttributeToResultObject, zap.String("key", k), zap.String("val", v)) }) diff --git a/uploader/multipart.go b/uploader/multipart.go index cda4b34..135ee88 100644 --- a/uploader/multipart.go +++ b/uploader/multipart.go @@ -3,6 +3,7 @@ package uploader import ( "io" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader/multipart" "go.uber.org/zap" ) @@ -28,7 +29,7 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF name := part.FormName() if name == "" { - l.Debug("ignore part, empty form name") + l.Debug(logs.IgnorePartEmptyFormName) continue } @@ -36,7 +37,7 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF // ignore multipart/form-data values if filename == "" { - l.Debug("ignore part, empty filename", zap.String("form", name)) + l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name)) continue } diff --git a/uploader/multipart_test.go b/uploader/multipart_test.go index aad2b66..d19cd5e 100644 --- a/uploader/multipart_test.go +++ b/uploader/multipart_test.go @@ -10,6 +10,7 @@ import ( "os" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -111,7 +112,7 @@ func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (Mul name := part.FormName() if name == "" { - l.Debug("ignore part, empty form name") + l.Debug(logs.IgnorePartEmptyFormName) continue } @@ -119,7 +120,7 @@ func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (Mul // ignore multipart/form-data values if filename == "" { - l.Debug("ignore part, empty filename", zap.String("form", name)) + l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name)) continue } diff --git a/uploader/upload.go b/uploader/upload.go index 57e426d..2832043 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -8,6 +8,7 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" @@ -77,7 +78,7 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { idCnr, err := utils.GetContainerID(ctx, scid, u.containerResolver) if err != nil { - log.Error("wrong container id", zap.Error(err)) + log.Error(logs.WrongContainerID, zap.Error(err)) response.Error(req, "wrong container id", fasthttp.StatusBadRequest) return } @@ -89,7 +90,7 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { } err := file.Close() log.Debug( - "close temporary multipart/form file", + logs.CloseTemporaryMultipartFormFile, zap.Stringer("address", addr), zap.String("filename", file.FileName()), zap.Error(err), @@ -97,13 +98,13 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { }() boundary := string(req.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(u.log, bodyStream, boundary); err != nil { - log.Error("could not receive multipart/form", zap.Error(err)) + log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) response.Error(req, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } filtered, err := filterHeaders(u.log, &req.Request.Header) if err != nil { - log.Error("could not process headers", zap.Error(err)) + log.Error(logs.CouldNotProcessHeaders, zap.Error(err)) response.Error(req, err.Error(), fasthttp.StatusBadRequest) return } @@ -111,14 +112,14 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { now := time.Now() if rawHeader := req.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { - log.Warn("could not parse client time", zap.String("Date header", string(rawHeader)), zap.Error(err)) + log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err)) } else { now = parsed } } if err = utils.PrepareExpirationHeader(req, u.pool, filtered, now); err != nil { - log.Error("could not prepare expiration header", zap.Error(err)) + log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) response.Error(req, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -170,7 +171,7 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(req); err != nil { - log.Error("could not encode response", zap.Error(err)) + log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) response.Error(req, "could not encode response", fasthttp.StatusBadRequest) return @@ -196,7 +197,7 @@ func (u *Uploader) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) { statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err) logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) - u.log.Error("could not store file in frostfs", logFields...) + u.log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) response.Error(r, msg, statusCode) } From 54eadc3c31c317779853b714f919394a233f8952 Mon Sep 17 00:00:00 2001 From: Dmitriy Zabolotskiy Date: Fri, 18 Aug 2023 17:21:59 +0300 Subject: [PATCH 410/548] [#69] Debian package: move home dir to match other components Signed-off-by: Dmitriy Zabolotskiy --- debian/frostfs-http-gw.postinst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/frostfs-http-gw.postinst b/debian/frostfs-http-gw.postinst index 360ceef..70deea5 100755 --- a/debian/frostfs-http-gw.postinst +++ b/debian/frostfs-http-gw.postinst @@ -21,7 +21,7 @@ set -e case "$1" in configure) USERNAME=http - id -u frostfs-$USERNAME >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /srv/frostfs_cache --system -M -U -c "FrostFS HTTP gateway" frostfs-$USERNAME + id -u frostfs-$USERNAME >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /var/lib/frostfs/$USERNAME --system -M -U -c "FrostFS HTTP gateway" frostfs-$USERNAME if ! dpkg-statoverride --list /etc/frostfs/$USERNAME >/dev/null; then chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/config.yaml || true @@ -29,7 +29,7 @@ case "$1" in chmod -f 0640 /etc/frostfs/$USERNAME/config.yaml || true fi USERDIR=$(getent passwd "frostfs-$USERNAME" | cut -d: -f6) - if ! dpkg-statoverride --list frostfs-"$USERDIR" >/dev/null; then + if ! dpkg-statoverride --list "$USERDIR" >/dev/null; then chown -f frostfs-$USERNAME: "$USERDIR" fi ;; From 7d47e88e36818a028ff56d9190a2e796c0adacd0 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 28 Aug 2023 14:09:40 +0300 Subject: [PATCH 411/548] [#76] Add go1.21 to CI Signed-off-by: Marina Biryukova --- .forgejo/workflows/builds.yml | 2 +- .forgejo/workflows/dco.yml | 2 +- .forgejo/workflows/tests.yml | 2 +- .forgejo/workflows/vulncheck.yml | 2 +- Dockerfile | 2 +- Makefile | 2 +- go.mod | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.forgejo/workflows/builds.yml b/.forgejo/workflows/builds.yml index 3df5081..aac6857 100644 --- a/.forgejo/workflows/builds.yml +++ b/.forgejo/workflows/builds.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.19', '1.20' ] + go_versions: [ '1.20', '1.21' ] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 6cbe4dd..3d38c4b 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.21' - name: Run commit format checker uses: https://git.frostfs.info/TrueCloudLab/dco-go@v1 diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 7a03020..d448a87 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.19', '1.20' ] + go_versions: [ '1.20', '1.21' ] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 0c9e908..0139e89 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.21' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/Dockerfile b/Dockerfile index 189dc22..8b450a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19-alpine as basebuilder +FROM golang:1.21-alpine as basebuilder RUN apk add --update make bash ca-certificates FROM basebuilder as builder diff --git a/Makefile b/Makefile index a4db526..6b6fa72 100755 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") -GO_VERSION ?= 1.19 +GO_VERSION ?= 1.20 LINT_VERSION ?= 1.49.0 BUILD ?= $(shell date -u --iso=seconds) diff --git a/go.mod b/go.mod index 632997f..6e6bc80 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw -go 1.19 +go 1.20 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 From dbc6804d273bf6b9df7b8f9a813e20e5c73835eb Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Tue, 29 Aug 2023 15:17:20 +0300 Subject: [PATCH 412/548] [#77] Add metrics for HTTP endpoint status Signed-off-by: Marina Biryukova --- app.go | 36 ++++++++++++++++++++++++++++-------- metrics/desc.go | 10 ++++++++++ metrics/metrics.go | 39 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 74 insertions(+), 11 deletions(-) diff --git a/app.go b/app.go index 864f39c..be9aaea 100644 --- a/app.go +++ b/app.go @@ -189,6 +189,13 @@ func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled b } } +func (m *gateMetrics) isEnabled() bool { + m.mu.RLock() + defer m.mu.RUnlock() + + return m.enabled +} + func (m *gateMetrics) SetEnabled(enabled bool) { if !enabled { m.logger.Warn(logs.MetricsAreDisabled) @@ -200,23 +207,17 @@ func (m *gateMetrics) SetEnabled(enabled bool) { } func (m *gateMetrics) SetHealth(status metrics.HealthStatus) { - m.mu.RLock() - if !m.enabled { - m.mu.RUnlock() + if !m.isEnabled() { return } - m.mu.RUnlock() m.provider.SetHealth(status) } func (m *gateMetrics) SetVersion(ver string) { - m.mu.RLock() - if !m.enabled { - m.mu.RUnlock() + if !m.isEnabled() { return } - m.mu.RUnlock() m.provider.SetVersion(ver) } @@ -231,6 +232,22 @@ func (m *gateMetrics) Shutdown() { m.mu.Unlock() } +func (m *gateMetrics) MarkHealthy(endpoint string) { + if !m.isEnabled() { + return + } + + m.provider.MarkHealthy(endpoint) +} + +func (m *gateMetrics) MarkUnhealthy(endpoint string) { + if !m.isEnabled() { + return + } + + m.provider.MarkUnhealthy(endpoint) +} + func remove(list []string, element string) []string { for i, item := range list { if item == element { @@ -327,6 +344,7 @@ func (a *app) Serve() { go func(i int) { a.log.Info(logs.StartingServer, zap.String("address", a.servers[i].Address())) if err := a.webServer.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed { + a.metrics.MarkUnhealthy(a.servers[i].Address()) a.log.Fatal(logs.ListenAndServe, zap.Error(err)) } }(i) @@ -508,9 +526,11 @@ func (a *app) initServers(ctx context.Context) { } srv, err := newServer(ctx, serverInfo) if err != nil { + a.metrics.MarkUnhealthy(serverInfo.Address) a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...) continue } + a.metrics.MarkHealthy(serverInfo.Address) a.servers = append(a.servers, srv) a.log.Info(logs.AddServer, fields...) diff --git a/metrics/desc.go b/metrics/desc.go index f2ff4f4..e10050c 100644 --- a/metrics/desc.go +++ b/metrics/desc.go @@ -66,6 +66,16 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"version"}, }, }, + serverSubsystem: { + healthMetric: Description{ + Type: dto.MetricType_GAUGE, + Namespace: namespace, + Subsystem: serverSubsystem, + Name: healthMetric, + Help: "HTTP Server endpoint health", + VariableLabels: []string{"endpoint"}, + }, + }, } type Description struct { diff --git a/metrics/metrics.go b/metrics/metrics.go index cf22099..bfb66ee 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -10,9 +10,10 @@ import ( ) const ( - namespace = "frostfs_http_gw" - stateSubsystem = "state" - poolSubsystem = "pool" + namespace = "frostfs_http_gw" + stateSubsystem = "state" + poolSubsystem = "pool" + serverSubsystem = "server" ) const ( @@ -60,9 +61,14 @@ type StatisticScraper interface { Statistic() pool.Statistic } +type serverMetrics struct { + endpointHealth *prometheus.GaugeVec +} + type GateMetrics struct { stateMetrics poolMetricsCollector + serverMetrics } type stateMetrics struct { @@ -87,15 +93,20 @@ func NewGateMetrics(p StatisticScraper) *GateMetrics { poolMetric := newPoolMetricsCollector(p) poolMetric.register() + serverMetric := newServerMetrics() + serverMetric.register() + return &GateMetrics{ stateMetrics: *stateMetric, poolMetricsCollector: *poolMetric, + serverMetrics: *serverMetric, } } func (g *GateMetrics) Unregister() { g.stateMetrics.unregister() prometheus.Unregister(&g.poolMetricsCollector) + g.serverMetrics.unregister() } func newStateMetrics() *stateMetrics { @@ -192,6 +203,28 @@ func (m *poolMetricsCollector) updateRequestsDuration(node pool.NodeStatistic) { m.requestDuration.WithLabelValues(node.Address(), methodCreateSession).Set(float64(node.AverageCreateSession().Milliseconds())) } +func newServerMetrics() *serverMetrics { + return &serverMetrics{ + endpointHealth: mustNewGaugeVec(appMetricsDesc[serverSubsystem][healthMetric]), + } +} + +func (m serverMetrics) register() { + prometheus.MustRegister(m.endpointHealth) +} + +func (m serverMetrics) unregister() { + prometheus.Unregister(m.endpointHealth) +} + +func (m serverMetrics) MarkHealthy(endpoint string) { + m.endpointHealth.WithLabelValues(endpoint).Set(float64(1)) +} + +func (m serverMetrics) MarkUnhealthy(endpoint string) { + m.endpointHealth.WithLabelValues(endpoint).Set(float64(0)) +} + // NewPrometheusService creates a new service for gathering prometheus metrics. func NewPrometheusService(log *zap.Logger, cfg Config) *Service { if log == nil { From 834d5b93e5d137b5847dbf7342ee806eb9cd5b80 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 1 Sep 2023 14:19:26 +0300 Subject: [PATCH 413/548] [#69] Fix postinstall script Post install script changes rights for user dir. With change of user dir (home dir), this dir isn't craeted anymore, so post install script fails. This commit changes useradd flag `-m` to create user dir. Signed-off-by: Alex Vanin --- debian/frostfs-http-gw.postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/frostfs-http-gw.postinst b/debian/frostfs-http-gw.postinst index 70deea5..1f25055 100755 --- a/debian/frostfs-http-gw.postinst +++ b/debian/frostfs-http-gw.postinst @@ -21,7 +21,7 @@ set -e case "$1" in configure) USERNAME=http - id -u frostfs-$USERNAME >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /var/lib/frostfs/$USERNAME --system -M -U -c "FrostFS HTTP gateway" frostfs-$USERNAME + id -u frostfs-$USERNAME >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /var/lib/frostfs/$USERNAME --system -m -U -c "FrostFS HTTP gateway" frostfs-$USERNAME if ! dpkg-statoverride --list /etc/frostfs/$USERNAME >/dev/null; then chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/config.yaml || true From 40568590c73c0ece76afa6fd61cde73ecdd6f7e1 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 31 Aug 2023 19:19:57 +0300 Subject: [PATCH 414/548] [#72] Support soft memory limit setting Signed-off-by: Roman Loginov --- app.go | 21 ++++++ config/config.env | 2 + config/config.yaml | 3 + docs/gate-configuration.md | 13 ++++ internal/logs/logs.go | 132 +++++++++++++++++++------------------ settings.go | 15 +++++ 6 files changed, 121 insertions(+), 65 deletions(-) diff --git a/app.go b/app.go index be9aaea..a0d6e0e 100644 --- a/app.go +++ b/app.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "os/signal" + "runtime/debug" "sync" "syscall" "time" @@ -128,6 +129,8 @@ func newApp(ctx context.Context, opt ...Option) App { user.IDFromKey(&owner, a.key.PrivateKey.PublicKey) a.owner = &owner + a.setRuntimeParameters() + a.initAppSettings() a.initResolver() a.initMetrics() @@ -407,6 +410,8 @@ func (a *app) configReload(ctx context.Context) { a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err)) } + a.setRuntimeParameters() + a.stopServices() a.startServices() @@ -596,3 +601,19 @@ func (a *app) initTracing(ctx context.Context) { a.log.Info(logs.TracingConfigUpdated) } } + +func (a *app) setRuntimeParameters() { + if len(os.Getenv("GOMEMLIMIT")) != 0 { + // default limit < yaml limit < app env limit < GOMEMLIMIT + a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) + return + } + + softMemoryLimit := fetchSoftMemoryLimit(a.cfg) + previous := debug.SetMemoryLimit(softMemoryLimit) + if softMemoryLimit != previous { + a.log.Info(logs.RuntimeSoftMemoryLimitUpdated, + zap.Int64("new_value", softMemoryLimit), + zap.Int64("old_value", previous)) + } +} diff --git a/config/config.env b/config/config.env index debdca2..62920a2 100644 --- a/config/config.env +++ b/config/config.env @@ -96,3 +96,5 @@ HTTP_GW_ZIP_COMPRESSION=false HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENDPOINT="localhost:4317" HTTP_GW_TRACING_EXPORTER="otlp_grpc" + +HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 diff --git a/config/config.yaml b/config/config.yaml index 510cb43..d2804d6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -101,3 +101,6 @@ pool_error_threshold: 100 # The number of errors on connection after which node zip: compression: false # Enable zip compression to download files by common prefix. + +runtime: + soft_memory_limit: 1gb diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 5b4edae..95e6c8e 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -53,6 +53,7 @@ $ cat http.log | `pprof` | [Pprof configuration](#pprof-section) | | `prometheus` | [Prometheus configuration](#prometheus-section) | | `tracing` | [Tracing configuration](#tracing-section) | +| `runtime` | [Runtime configuration](#runtime-section) | # General section @@ -256,3 +257,15 @@ tracing: | `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | | `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | | `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | + +# `runtime` section +Contains runtime parameters. + +```yaml +runtime: + soft_memory_limit: 1gb +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `soft_memory_limit` | `size` | yes | maxint64 | Soft memory limit for the runtime. Zero or no value stands for no limit. If `GOMEMLIMIT` environment variable is set, the value from the configuration file will be ignored. | \ No newline at end of file diff --git a/internal/logs/logs.go b/internal/logs/logs.go index d47a2c3..ebb3c24 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -1,69 +1,71 @@ package logs const ( - CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/* - CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go - CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go - WrongContainerID = "wrong container id" // Error in ../../downloader/download.go and uploader/upload.go - WrongObjectID = "wrong object id" // Error in ../../downloader/download.go - ObjectWasntFound = "object wasn't found" // Error in ../../downloader/download.go - ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go - CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go - ObjectNotFound = "object not found" // Error in ../../downloader/download.go - ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go - CouldNotCheckContainerExistence = "could not check container existence" // Error in ../../downloader/download.go - FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go - IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go - ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go - CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go - ServiceIsRunning = "service is running" // Info in ../../metrics/service.go - ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go - ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go - ShuttingDownService = "shutting down service" // Info in ../../metrics/service.go - CantShutDownService = "can't shut down service" // Panic in ../../metrics/service.go - IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go - IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go - CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go - CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go - CouldNotProcessHeaders = "could not process headers" // Error in ../../uploader/upload.go - CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go - CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go - CouldNotEncodeResponse = "could not encode response" // Error in ../../uploader/upload.go - CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go - AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go - FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go - ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go - MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go - NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" // Info in ../../app.go - StartingApplication = "starting application" // Info in ../../app.go - StartingServer = "starting server" // Info in ../../app.go - ListenAndServe = "listen and serve" // Fatal in ../../app.go - ShuttingDownWebServer = "shutting down web server" // Info in ../../app.go - FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../app.go - SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../app.go - FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../app.go - FailedToReloadConfig = "failed to reload config" // Warn in ../../app.go - LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../app.go - FailedToUpdateResolvers = "failed to update resolvers" // Warn in ../../app.go - FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../app.go - SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../app.go - AddedPathUploadCid = "added path /upload/{cid}" // Info in ../../app.go - AddedPathGetCidOid = "added path /get/{cid}/{oid}" // Info in ../../app.go - AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" // Info in ../../app.go - AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" // Info in ../../app.go - Request = "request" // Info in ../../app.go - CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" // Error in ../../app.go - FailedToAddServer = "failed to add server" // Warn in ../../app.go - AddServer = "add server" // Info in ../../app.go - NoHealthyServers = "no healthy servers" // Fatal in ../../app.go - FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../app.go - TracingConfigUpdated = "tracing config updated" // Info in ../../app.go - ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" // Warn in ../../app.go - CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../settings.go - UsingCredentials = "using credentials" // Info in ../../settings.go - FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../settings.go - FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../settings.go - FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../settings.go - FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../settings.go - AddedStoragePeer = "added storage peer" // Info in ../../settings.go + CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/* + CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go + CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go + WrongContainerID = "wrong container id" // Error in ../../downloader/download.go and uploader/upload.go + WrongObjectID = "wrong object id" // Error in ../../downloader/download.go + ObjectWasntFound = "object wasn't found" // Error in ../../downloader/download.go + ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go + CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go + ObjectNotFound = "object not found" // Error in ../../downloader/download.go + ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go + CouldNotCheckContainerExistence = "could not check container existence" // Error in ../../downloader/download.go + FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go + IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go + ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go + CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go + ServiceIsRunning = "service is running" // Info in ../../metrics/service.go + ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go + ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go + ShuttingDownService = "shutting down service" // Info in ../../metrics/service.go + CantShutDownService = "can't shut down service" // Panic in ../../metrics/service.go + IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go + IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go + CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go + CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go + CouldNotProcessHeaders = "could not process headers" // Error in ../../uploader/upload.go + CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go + CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go + CouldNotEncodeResponse = "could not encode response" // Error in ../../uploader/upload.go + CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go + AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go + FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go + ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go + MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go + NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" // Info in ../../app.go + StartingApplication = "starting application" // Info in ../../app.go + StartingServer = "starting server" // Info in ../../app.go + ListenAndServe = "listen and serve" // Fatal in ../../app.go + ShuttingDownWebServer = "shutting down web server" // Info in ../../app.go + FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../app.go + SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../app.go + FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../app.go + FailedToReloadConfig = "failed to reload config" // Warn in ../../app.go + LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../app.go + FailedToUpdateResolvers = "failed to update resolvers" // Warn in ../../app.go + FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../app.go + SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../app.go + AddedPathUploadCid = "added path /upload/{cid}" // Info in ../../app.go + AddedPathGetCidOid = "added path /get/{cid}/{oid}" // Info in ../../app.go + AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" // Info in ../../app.go + AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" // Info in ../../app.go + Request = "request" // Info in ../../app.go + CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" // Error in ../../app.go + FailedToAddServer = "failed to add server" // Warn in ../../app.go + AddServer = "add server" // Info in ../../app.go + NoHealthyServers = "no healthy servers" // Fatal in ../../app.go + FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../app.go + TracingConfigUpdated = "tracing config updated" // Info in ../../app.go + ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" // Warn in ../../app.go + RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" // Warn in ../../app.go + RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" // Info in ../../app.go + CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../settings.go + UsingCredentials = "using credentials" // Info in ../../settings.go + FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../settings.go + FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../settings.go + FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../settings.go + FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../settings.go + AddedStoragePeer = "added storage peer" // Info in ../../settings.go ) diff --git a/settings.go b/settings.go index ed13da2..6708ad4 100644 --- a/settings.go +++ b/settings.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "math" "os" "path" "runtime" @@ -36,6 +37,8 @@ const ( defaultPoolErrorThreshold uint32 = 100 + defaultSoftMemoryLimit = math.MaxInt64 + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" @@ -90,6 +93,9 @@ const ( // Zip compression. cfgZipCompression = "zip.compression" + // Runtime. + cfgSoftMemoryLimit = "runtime.soft_memory_limit" + // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -516,3 +522,12 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam { return nodes } + +func fetchSoftMemoryLimit(cfg *viper.Viper) int64 { + softMemoryLimit := cfg.GetSizeInBytes(cfgSoftMemoryLimit) + if softMemoryLimit <= 0 { + softMemoryLimit = defaultSoftMemoryLimit + } + + return int64(softMemoryLimit) +} From add07a21ed4f4aa47649c2e97b706880afeec51d Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 4 Sep 2023 16:09:57 +0300 Subject: [PATCH 415/548] [#71] Add log constants linter Signed-off-by: Marina Biryukova --- .forgejo/workflows/tests.yml | 13 ++++++++++--- .golangci.yml | 11 +++++++++++ .pre-commit-config.yaml | 24 ++++++++++++++++++------ Makefile | 23 +++++++++++++++++++++-- 4 files changed, 60 insertions(+), 11 deletions(-) diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index d448a87..14b9edf 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -7,10 +7,17 @@ jobs: steps: - uses: actions/checkout@v3 - - name: golangci-lint - uses: https://github.com/golangci/golangci-lint-action@v2 + - name: Set up Go + uses: actions/setup-go@v3 with: - version: latest + go-version: '1.21' + cache: true + + - name: Install linters + run: make lint-install + + - name: Run linters + run: make lint tests: name: Tests diff --git a/.golangci.yml b/.golangci.yml index a271450..5459bde 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -24,6 +24,16 @@ linters-settings: govet: # report about shadowed variables check-shadowing: false + custom: + truecloudlab-linters: + path: bin/external_linters.so + original-url: git.frostfs.info/TrueCloudLab/linters.git + settings: + noliteral: + enable: true + target-methods: ["Fatal"] + disable-packages: ["req", "r"] + constants-package: "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" linters: enable: @@ -45,6 +55,7 @@ linters: - gofmt - whitespace - goimports + - truecloudlab-linters disable-all: true fast: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4fde2a0..e97fc23 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,9 +37,21 @@ repos: - repo: local hooks: - - id: go-unit-tests - name: go unit tests - entry: make test - pass_filenames: false - types: [go] - language: system + - id: make-lint-install + name: install linters + entry: make lint-install + language: system + pass_filenames: false + + - id: make-lint + name: run linters + entry: make lint + language: system + pass_filenames: false + + - id: go-unit-tests + name: go unit tests + entry: make test + pass_filenames: false + types: [go] + language: system diff --git a/Makefile b/Makefile index 6b6fa72..c5296e1 100755 --- a/Makefile +++ b/Makefile @@ -3,7 +3,8 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") GO_VERSION ?= 1.20 -LINT_VERSION ?= 1.49.0 +LINT_VERSION ?= 1.54.0 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.2 BUILD ?= $(shell date -u --iso=seconds) HUB_IMAGE ?= truecloudlab/frostfs-http-gw @@ -11,6 +12,10 @@ HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" METRICS_DUMP_OUT ?= ./metrics-dump.json +OUTPUT_LINT_DIR ?= $(shell pwd)/bin +LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION) +TMP_DIR := .cache + # List of binaries to build. For now just one. BINDIR = bin DIRS = $(BINDIR) @@ -103,9 +108,23 @@ dirty-image: -f Dockerfile.dirty \ -t $(HUB_IMAGE)-dirty:$(HUB_TAG) . +# Install linters + lint-install: + @mkdir -p $(TMP_DIR) + @rm -rf $(TMP_DIR)/linters + @git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters + @@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR) + @rm -rf $(TMP_DIR)/linters + @rmdir $(TMP_DIR) 2>/dev/null || true + @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) + # Run linters lint: - @golangci-lint --timeout=5m run + @if [ ! -d "$(LINT_DIR)" ]; then \ + echo "Run make lint-install"; \ + exit 1; \ + fi + $(LINT_DIR)/golangci-lint --timeout=5m run # Run linters in Docker docker/lint: From d2199435426b673f89fef39e7234061bd476caad Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Thu, 31 Aug 2023 11:37:03 +0300 Subject: [PATCH 416/548] [#73] Uploader, downloader structures refactoring Signed-off-by: Marina Biryukova --- Dockerfile => .docker/Dockerfile | 0 Dockerfile.dirty => .docker/Dockerfile.dirty | 0 Makefile | 10 +- app.go => cmd/http-gw/app.go | 39 +- .../http-gw/integration_test.go | 2 +- main.go => cmd/http-gw/main.go | 0 misc.go => cmd/http-gw/misc.go | 0 server.go => cmd/http-gw/server.go | 0 settings.go => cmd/http-gw/settings.go | 0 downloader/download.go | 537 ------------------ go.mod | 2 +- {api => internal/api}/layer/tree_service.go | 2 +- {api => internal/api}/tree.go | 0 internal/handler/download.go | 210 +++++++ {uploader => internal/handler}/filter.go | 2 +- {uploader => internal/handler}/filter_test.go | 2 +- internal/handler/handler.go | 193 +++++++ {downloader => internal/handler}/head.go | 18 +- {uploader => internal/handler}/multipart.go | 4 +- .../handler}/multipart/multipart.go | 0 .../handler}/multipart_test.go | 2 +- internal/handler/reader.go | 141 +++++ .../handler}/reader_test.go | 2 +- {uploader => internal/handler}/upload.go | 93 +-- internal/handler/utils.go | 60 ++ tree/tree.go | 4 +- utils/util.go | 13 - 27 files changed, 672 insertions(+), 664 deletions(-) rename Dockerfile => .docker/Dockerfile (100%) rename Dockerfile.dirty => .docker/Dockerfile.dirty (100%) rename app.go => cmd/http-gw/app.go (91%) rename integration_test.go => cmd/http-gw/integration_test.go (99%) rename main.go => cmd/http-gw/main.go (100%) rename misc.go => cmd/http-gw/misc.go (100%) rename server.go => cmd/http-gw/server.go (100%) rename settings.go => cmd/http-gw/settings.go (100%) delete mode 100644 downloader/download.go rename {api => internal/api}/layer/tree_service.go (90%) rename {api => internal/api}/tree.go (100%) create mode 100644 internal/handler/download.go rename {uploader => internal/handler}/filter.go (98%) rename {uploader => internal/handler}/filter_test.go (98%) create mode 100644 internal/handler/handler.go rename {downloader => internal/handler}/head.go (86%) rename {uploader => internal/handler}/multipart.go (92%) rename {uploader => internal/handler}/multipart/multipart.go (100%) rename {uploader => internal/handler}/multipart_test.go (99%) create mode 100644 internal/handler/reader.go rename {downloader => internal/handler}/reader_test.go (98%) rename {uploader => internal/handler}/upload.go (73%) create mode 100644 internal/handler/utils.go diff --git a/Dockerfile b/.docker/Dockerfile similarity index 100% rename from Dockerfile rename to .docker/Dockerfile diff --git a/Dockerfile.dirty b/.docker/Dockerfile.dirty similarity index 100% rename from Dockerfile.dirty rename to .docker/Dockerfile.dirty diff --git a/Makefile b/Makefile index c5296e1..d02d41b 100755 --- a/Makefile +++ b/Makefile @@ -18,8 +18,8 @@ TMP_DIR := .cache # List of binaries to build. For now just one. BINDIR = bin -DIRS = $(BINDIR) -BINS = $(BINDIR)/frostfs-http-gw +CMDS = $(addprefix frostfs-, $(notdir $(wildcard cmd/*))) +BINS = $(addprefix $(BINDIR)/, $(CMDS)) .PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean @@ -37,7 +37,7 @@ $(BINS): $(DIRS) dep CGO_ENABLED=0 \ go build -v -trimpath \ -ldflags "-X main.Version=$(VERSION)" \ - -o $@ ./ + -o $@ ./cmd/$(subst frostfs-,,$(notdir $@)) $(DIRS): @echo "⇒ Ensure dir: $@" @@ -90,7 +90,7 @@ image: --build-arg REPO=$(REPO) \ --build-arg VERSION=$(VERSION) \ --rm \ - -f Dockerfile \ + -f .docker/Dockerfile \ -t $(HUB_IMAGE):$(HUB_TAG) . # Push Docker image to the hub @@ -105,7 +105,7 @@ dirty-image: --build-arg REPO=$(REPO) \ --build-arg VERSION=$(VERSION) \ --rm \ - -f Dockerfile.dirty \ + -f .docker/Dockerfile.dirty \ -t $(HUB_IMAGE)-dirty:$(HUB_TAG) . # Install linters diff --git a/app.go b/cmd/http-gw/app.go similarity index 91% rename from app.go rename to cmd/http-gw/app.go index a0d6e0e..40db433 100644 --- a/app.go +++ b/cmd/http-gw/app.go @@ -11,15 +11,14 @@ import ( "syscall" "time" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/downloader" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" @@ -51,15 +50,10 @@ type ( resolver *resolver.ContainerResolver metrics *gateMetrics services []*metrics.Service - settings *appSettings + settings *handler.Settings servers []Server } - appSettings struct { - Uploader *uploader.Settings - Downloader *downloader.Settings - } - // App is an interface for the main gateway function. App interface { Wait() @@ -140,10 +134,7 @@ func newApp(ctx context.Context, opt ...Option) App { } func (a *app) initAppSettings() { - a.settings = &appSettings{ - Uploader: &uploader.Settings{}, - Downloader: &downloader.Settings{}, - } + a.settings = &handler.Settings{} a.updateSettings() } @@ -334,11 +325,10 @@ func (a *app) setHealthStatus() { } func (a *app) Serve() { - uploadRoutes := uploader.New(a.AppParams(), a.settings.Uploader) - downloadRoutes := downloader.New(a.AppParams(), a.settings.Downloader, tree.NewTree(services.NewPoolWrapper(a.treePool))) + handler := handler.New(a.AppParams(), a.settings, tree.NewTree(services.NewPoolWrapper(a.treePool))) // Configure router. - a.configureRouter(uploadRoutes, downloadRoutes) + a.configureRouter(handler) a.startServices() a.initServers(a.ctx) @@ -425,8 +415,8 @@ func (a *app) configReload(ctx context.Context) { } func (a *app) updateSettings() { - a.settings.Uploader.SetDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) - a.settings.Downloader.SetZipCompression(a.cfg.GetBool(cfgZipCompression)) + a.settings.SetDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) + a.settings.SetZipCompression(a.cfg.GetBool(cfgZipCompression)) } func (a *app) startServices() { @@ -450,7 +440,7 @@ func (a *app) stopServices() { } } -func (a *app) configureRouter(uploadRoutes *uploader.Uploader, downloadRoutes *downloader.Downloader) { +func (a *app) configureRouter(handler *handler.Handler) { r := router.New() r.RedirectTrailingSlash = true r.NotFound = func(r *fasthttp.RequestCtx) { @@ -459,15 +449,16 @@ func (a *app) configureRouter(uploadRoutes *uploader.Uploader, downloadRoutes *d r.MethodNotAllowed = func(r *fasthttp.RequestCtx) { response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.logger(a.tokenizer(a.tracer(uploadRoutes.Upload)))) + + r.POST("/upload/{cid}", a.logger(a.tokenizer(a.tracer(handler.Upload)))) a.log.Info(logs.AddedPathUploadCid) - r.GET("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.DownloadByAddressOrBucketName)))) - r.HEAD("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.HeadByAddressOrBucketName)))) + r.GET("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(handler.DownloadByAddressOrBucketName)))) + r.HEAD("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(handler.HeadByAddressOrBucketName)))) a.log.Info(logs.AddedPathGetCidOid) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.DownloadByAttribute)))) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.HeadByAttribute)))) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(handler.DownloadByAttribute)))) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(handler.HeadByAttribute)))) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.logger(a.tokenizer(a.tracer(downloadRoutes.DownloadZipped)))) + r.GET("/zip/{cid}/{prefix:*}", a.logger(a.tokenizer(a.tracer(handler.DownloadZipped)))) a.log.Info(logs.AddedPathZipCidPrefix) a.webServer.Handler = r.Handler diff --git a/integration_test.go b/cmd/http-gw/integration_test.go similarity index 99% rename from integration_test.go rename to cmd/http-gw/integration_test.go index 34506c4..76a8325 100644 --- a/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -83,7 +83,7 @@ func runServer() (App, context.CancelFunc) { v := getDefaultConfig() l, lvl := newLogger(v) application := newApp(cancelCtx, WithConfig(v), WithLogger(l, lvl)) - go application.Serve(cancelCtx) + go application.Serve() return application, cancel } diff --git a/main.go b/cmd/http-gw/main.go similarity index 100% rename from main.go rename to cmd/http-gw/main.go diff --git a/misc.go b/cmd/http-gw/misc.go similarity index 100% rename from misc.go rename to cmd/http-gw/misc.go diff --git a/server.go b/cmd/http-gw/server.go similarity index 100% rename from server.go rename to cmd/http-gw/server.go diff --git a/settings.go b/cmd/http-gw/settings.go similarity index 100% rename from settings.go rename to cmd/http-gw/settings.go diff --git a/downloader/download.go b/downloader/download.go deleted file mode 100644 index cd7f72f..0000000 --- a/downloader/download.go +++ /dev/null @@ -1,537 +0,0 @@ -package downloader - -import ( - "archive/zip" - "bufio" - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "path" - "strconv" - "strings" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" - "github.com/valyala/fasthttp" - "go.uber.org/atomic" - "go.uber.org/zap" -) - -type request struct { - *fasthttp.RequestCtx - log *zap.Logger -} - -func isValidToken(s string) bool { - for _, c := range s { - if c <= ' ' || c > 127 { - return false - } - if strings.ContainsRune("()<>@,;:\\\"/[]?={}", c) { - return false - } - } - return true -} - -func isValidValue(s string) bool { - for _, c := range s { - // HTTP specification allows for more technically, but we don't want to escape things. - if c < ' ' || c > 127 || c == '"' { - return false - } - } - return true -} - -type readCloser struct { - io.Reader - io.Closer -} - -// initializes io.Reader with the limited size and detects Content-Type from it. -// Returns r's error directly. Also returns the processed data. -func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (string, []byte, error) { - if maxSize > sizeToDetectType { - maxSize = sizeToDetectType - } - - buf := make([]byte, maxSize) // maybe sync-pool the slice? - - r, err := rInit(maxSize) - if err != nil { - return "", nil, err - } - - n, err := r.Read(buf) - if err != nil && err != io.EOF { - return "", nil, err - } - - buf = buf[:n] - - return http.DetectContentType(buf), buf, err // to not lose io.EOF -} - -func receiveFile(ctx context.Context, req request, clnt *pool.Pool, objectAddress oid.Address) { - var ( - err error - dis = "inline" - start = time.Now() - filename string - ) - - var prm pool.PrmObjectGet - prm.SetAddress(objectAddress) - if btoken := bearerToken(ctx); btoken != nil { - prm.UseBearer(*btoken) - } - - rObj, err := clnt.GetObject(ctx, prm) - if err != nil { - req.handleFrostFSErr(err, start) - return - } - - // we can't close reader in this function, so how to do it? - - if req.Request.URI().QueryArgs().GetBool("download") { - dis = "attachment" - } - - payloadSize := rObj.Header.PayloadSize() - - req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) - var contentType string - for _, attr := range rObj.Header.Attributes() { - key := attr.Key() - val := attr.Value() - if !isValidToken(key) || !isValidValue(val) { - continue - } - - key = utils.BackwardTransformIfSystem(key) - - req.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) - switch key { - case object.AttributeFileName: - filename = val - case object.AttributeTimestamp: - value, err := strconv.ParseInt(val, 10, 64) - if err != nil { - req.log.Info(logs.CouldntParseCreationDate, - zap.String("key", key), - zap.String("val", val), - zap.Error(err)) - continue - } - req.Response.Header.Set(fasthttp.HeaderLastModified, - time.Unix(value, 0).UTC().Format(http.TimeFormat)) - case object.AttributeContentType: - contentType = val - } - } - - idsToResponse(&req.Response, &rObj.Header) - - if len(contentType) == 0 { - // determine the Content-Type from the payload head - var payloadHead []byte - - contentType, payloadHead, err = readContentType(payloadSize, func(uint64) (io.Reader, error) { - return rObj.Payload, nil - }) - if err != nil && err != io.EOF { - req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) - response.Error(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) - return - } - - // reset payload reader since a part of the data has been read - var headReader io.Reader = bytes.NewReader(payloadHead) - - if err != io.EOF { // otherwise, we've already read full payload - headReader = io.MultiReader(headReader, rObj.Payload) - } - - // note: we could do with io.Reader, but SetBodyStream below closes body stream - // if it implements io.Closer and that's useful for us. - rObj.Payload = readCloser{headReader, rObj.Payload} - } - req.SetContentType(contentType) - - req.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) - - req.Response.SetBodyStream(rObj.Payload, int(payloadSize)) -} - -func bearerToken(ctx context.Context) *bearer.Token { - if tkn, err := tokens.LoadBearerToken(ctx); err == nil { - return tkn - } - return nil -} - -func (r *request) handleFrostFSErr(err error, start time.Time) { - logFields := []zap.Field{ - zap.Stringer("elapsed", time.Since(start)), - zap.Error(err), - } - statusCode, msg, additionalFields := response.FormErrorResponse("could not receive object", err) - logFields = append(logFields, additionalFields...) - - r.log.Error(logs.CouldNotReceiveObject, logFields...) - response.Error(r.RequestCtx, msg, statusCode) -} - -// Downloader is a download request handler. -type Downloader struct { - log *zap.Logger - pool *pool.Pool - containerResolver *resolver.ContainerResolver - settings *Settings - tree *tree.Tree -} - -// Settings stores reloading parameters, so it has to provide atomic getters and setters. -type Settings struct { - zipCompression atomic.Bool -} - -func (s *Settings) ZipCompression() bool { - return s.zipCompression.Load() -} - -func (s *Settings) SetZipCompression(val bool) { - s.zipCompression.Store(val) -} - -// New creates an instance of Downloader using specified options. -func New(params *utils.AppParams, settings *Settings, tree *tree.Tree) *Downloader { - return &Downloader{ - log: params.Logger, - pool: params.Pool, - settings: settings, - containerResolver: params.Resolver, - tree: tree, - } -} - -func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { - return &request{ - RequestCtx: ctx, - log: log, - } -} - -// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. -func (d *Downloader) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { - test, _ := c.UserValue("oid").(string) - var id oid.ID - err := id.DecodeString(test) - if err != nil { - d.byBucketname(c, receiveFile) - } else { - d.byAddress(c, receiveFile) - } -} - -// byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that -// prepares request and object address to it. -func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, *pool.Pool, oid.Address)) { - var ( - idCnr, _ = c.UserValue("cid").(string) - idObj, _ = c.UserValue("oid").(string) - log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj)) - ) - - ctx := utils.GetContextFromRequest(c) - - cnrID, err := utils.GetContainerID(ctx, idCnr, d.containerResolver) - if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(c, "wrong container id", fasthttp.StatusBadRequest) - return - } - - objID := new(oid.ID) - if err = objID.DecodeString(idObj); err != nil { - log.Error(logs.WrongObjectID, zap.Error(err)) - response.Error(c, "wrong object id", fasthttp.StatusBadRequest) - return - } - - var addr oid.Address - addr.SetContainer(*cnrID) - addr.SetObject(*objID) - - f(ctx, *d.newRequest(c, log), d.pool, addr) -} - -// byBucketname is a wrapper for function (e.g. request.headObject, request.receiveFile) that -// prepares request and object address to it. -func (d *Downloader) byBucketname(req *fasthttp.RequestCtx, f func(context.Context, request, *pool.Pool, oid.Address)) { - var ( - bucketname = req.UserValue("cid").(string) - key = req.UserValue("oid").(string) - log = d.log.With(zap.String("bucketname", bucketname), zap.String("key", key)) - ) - - ctx := utils.GetContextFromRequest(req) - - cnrID, err := utils.GetContainerID(ctx, bucketname, d.containerResolver) - if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(req, "wrong container id", fasthttp.StatusBadRequest) - return - } - - foundOid, err := d.tree.GetLatestVersion(ctx, cnrID, key) - if err != nil { - log.Error(logs.ObjectWasntFound, zap.Error(err)) - response.Error(req, "object wasn't found", fasthttp.StatusNotFound) - return - } - if foundOid.DeleteMarker { - log.Error(logs.ObjectWasDeleted) - response.Error(req, "object deleted", fasthttp.StatusNotFound) - return - } - - var addr oid.Address - addr.SetContainer(*cnrID) - addr.SetObject(foundOid.OID) - - f(ctx, *d.newRequest(req, log), d.pool, addr) -} - -// DownloadByAttribute handles attribute-based download requests. -func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { - d.byAttribute(c, receiveFile) -} - -// byAttribute is a wrapper similar to byAddress. -func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, *pool.Pool, oid.Address)) { - var ( - scid, _ = c.UserValue("cid").(string) - key, _ = url.QueryUnescape(c.UserValue("attr_key").(string)) - val, _ = url.QueryUnescape(c.UserValue("attr_val").(string)) - log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) - ) - - ctx := utils.GetContextFromRequest(c) - - containerID, err := utils.GetContainerID(ctx, scid, d.containerResolver) - if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(c, "wrong container id", fasthttp.StatusBadRequest) - return - } - - res, err := d.search(ctx, containerID, key, val, object.MatchStringEqual) - if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) - response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) - return - } - - defer res.Close() - - buf := make([]oid.ID, 1) - - n, err := res.Read(buf) - if n == 0 { - if errors.Is(err, io.EOF) { - log.Error(logs.ObjectNotFound, zap.Error(err)) - response.Error(c, "object not found", fasthttp.StatusNotFound) - return - } - - log.Error(logs.ReadObjectListFailed, zap.Error(err)) - response.Error(c, "read object list failed: "+err.Error(), fasthttp.StatusBadRequest) - return - } - - var addrObj oid.Address - addrObj.SetContainer(*containerID) - addrObj.SetObject(buf[0]) - - f(ctx, *d.newRequest(c, log), d.pool, addrObj) -} - -func (d *Downloader) search(ctx context.Context, cid *cid.ID, key, val string, op object.SearchMatchType) (pool.ResObjectSearch, error) { - filters := object.NewSearchFilters() - filters.AddRootFilter() - filters.AddFilter(key, val, op) - - var prm pool.PrmObjectSearch - prm.SetContainerID(*cid) - prm.SetFilters(filters) - if btoken := bearerToken(ctx); btoken != nil { - prm.UseBearer(*btoken) - } - - return d.pool.SearchObjects(ctx, prm) -} - -func (d *Downloader) getContainer(ctx context.Context, cnrID cid.ID) (container.Container, error) { - var prm pool.PrmContainerGet - prm.SetContainerID(cnrID) - - return d.pool.GetContainer(ctx, prm) -} - -func (d *Downloader) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { - method := zip.Store - if d.settings.ZipCompression() { - method = zip.Deflate - } - - filePath := getZipFilePath(obj) - if len(filePath) == 0 || filePath[len(filePath)-1] == '/' { - return nil, fmt.Errorf("invalid filepath '%s'", filePath) - } - - return zw.CreateHeader(&zip.FileHeader{ - Name: filePath, - Method: method, - Modified: time.Now(), - }) -} - -// DownloadZipped handles zip by prefix requests. -func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { - scid, _ := c.UserValue("cid").(string) - prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string)) - log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) - - ctx := utils.GetContextFromRequest(c) - - containerID, err := utils.GetContainerID(ctx, scid, d.containerResolver) - if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(c, "wrong container id", fasthttp.StatusBadRequest) - return - } - - // check if container exists here to be able to return 404 error, - // otherwise we get this error only in object iteration step - // and client get 200 OK. - if _, err = d.getContainer(ctx, *containerID); err != nil { - log.Error(logs.CouldNotCheckContainerExistence, zap.Error(err)) - if client.IsErrContainerNotFound(err) { - response.Error(c, "Not Found", fasthttp.StatusNotFound) - return - } - response.Error(c, "could not check container existence: "+err.Error(), fasthttp.StatusBadRequest) - return - } - - resSearch, err := d.search(ctx, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) - if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) - response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) - return - } - - c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") - c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") - c.Response.SetStatusCode(http.StatusOK) - - c.SetBodyStreamWriter(func(w *bufio.Writer) { - defer resSearch.Close() - - zipWriter := zip.NewWriter(w) - - var bufZip []byte - var addr oid.Address - - empty := true - called := false - btoken := bearerToken(ctx) - addr.SetContainer(*containerID) - - errIter := resSearch.Iterate(func(id oid.ID) bool { - called = true - - if empty { - bufZip = make([]byte, 3<<20) // the same as for upload - } - empty = false - - addr.SetObject(id) - if err = d.zipObject(ctx, zipWriter, addr, btoken, bufZip); err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.String("oid", id.EncodeToString()), zap.Error(err)) - } - - return false - }) - if errIter != nil { - log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) - } else if !called { - log.Error(logs.ObjectsNotFound) - } - - if err = zipWriter.Close(); err != nil { - log.Error(logs.CloseZipWriter, zap.Error(err)) - } - }) -} - -func (d *Downloader) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error { - var prm pool.PrmObjectGet - prm.SetAddress(addr) - if btoken != nil { - prm.UseBearer(*btoken) - } - - resGet, err := d.pool.GetObject(ctx, prm) - if err != nil { - return fmt.Errorf("get FrostFS object: %v", err) - } - - objWriter, err := d.addObjectToZip(zipWriter, &resGet.Header) - if err != nil { - return fmt.Errorf("zip create header: %v", err) - } - - if _, err = io.CopyBuffer(objWriter, resGet.Payload, bufZip); err != nil { - return fmt.Errorf("copy object payload to zip file: %v", err) - } - - if err = resGet.Payload.Close(); err != nil { - return fmt.Errorf("object body close error: %w", err) - } - - if err = zipWriter.Flush(); err != nil { - return fmt.Errorf("flush zip writer: %v", err) - } - - return nil -} - -func getZipFilePath(obj *object.Object) string { - for _, attr := range obj.Attributes() { - if attr.Key() == object.AttributeFilePath { - return attr.Value() - } - } - - return "" -} diff --git a/go.mod b/go.mod index 6e6bc80..80d794c 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/valyala/fasthttp v1.34.0 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 - go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 google.golang.org/grpc v1.55.0 ) @@ -97,6 +96,7 @@ require ( go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/otel/sdk v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.9.0 // indirect golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect diff --git a/api/layer/tree_service.go b/internal/api/layer/tree_service.go similarity index 90% rename from api/layer/tree_service.go rename to internal/api/layer/tree_service.go index 9852257..beb1e7a 100644 --- a/api/layer/tree_service.go +++ b/internal/api/layer/tree_service.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) diff --git a/api/tree.go b/internal/api/tree.go similarity index 100% rename from api/tree.go rename to internal/api/tree.go diff --git a/internal/handler/download.go b/internal/handler/download.go new file mode 100644 index 0000000..8ee76bf --- /dev/null +++ b/internal/handler/download.go @@ -0,0 +1,210 @@ +package handler + +import ( + "archive/zip" + "bufio" + "context" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. +func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { + test, _ := c.UserValue("oid").(string) + var id oid.ID + err := id.DecodeString(test) + if err != nil { + h.byBucketname(c, h.receiveFile) + } else { + h.byAddress(c, h.receiveFile) + } +} + +func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { + return &request{ + RequestCtx: ctx, + log: log, + } +} + +// DownloadByAttribute handles attribute-based download requests. +func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { + h.byAttribute(c, h.receiveFile) +} + +func (h *Handler) search(ctx context.Context, cid *cid.ID, key, val string, op object.SearchMatchType) (pool.ResObjectSearch, error) { + filters := object.NewSearchFilters() + filters.AddRootFilter() + filters.AddFilter(key, val, op) + + var prm pool.PrmObjectSearch + prm.SetContainerID(*cid) + prm.SetFilters(filters) + if btoken := bearerToken(ctx); btoken != nil { + prm.UseBearer(*btoken) + } + + return h.pool.SearchObjects(ctx, prm) +} + +func (h *Handler) getContainer(ctx context.Context, cnrID cid.ID) (container.Container, error) { + var prm pool.PrmContainerGet + prm.SetContainerID(cnrID) + + return h.pool.GetContainer(ctx, prm) +} + +func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { + method := zip.Store + if h.settings.ZipCompression() { + method = zip.Deflate + } + + filePath := getZipFilePath(obj) + if len(filePath) == 0 || filePath[len(filePath)-1] == '/' { + return nil, fmt.Errorf("invalid filepath '%s'", filePath) + } + + return zw.CreateHeader(&zip.FileHeader{ + Name: filePath, + Method: method, + Modified: time.Now(), + }) +} + +// DownloadZipped handles zip by prefix requests. +func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { + scid, _ := c.UserValue("cid").(string) + prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string)) + log := h.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) + + ctx := utils.GetContextFromRequest(c) + + containerID, err := h.getContainerID(ctx, scid) + if err != nil { + log.Error(logs.WrongContainerID, zap.Error(err)) + response.Error(c, "wrong container id", fasthttp.StatusBadRequest) + return + } + + // check if container exists here to be able to return 404 error, + // otherwise we get this error only in object iteration step + // and client get 200 OK. + if _, err = h.getContainer(ctx, *containerID); err != nil { + log.Error(logs.CouldNotCheckContainerExistence, zap.Error(err)) + if client.IsErrContainerNotFound(err) { + response.Error(c, "Not Found", fasthttp.StatusNotFound) + return + } + response.Error(c, "could not check container existence: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + resSearch, err := h.search(ctx, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) + if err != nil { + log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") + c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") + c.Response.SetStatusCode(http.StatusOK) + + c.SetBodyStreamWriter(func(w *bufio.Writer) { + defer resSearch.Close() + + zipWriter := zip.NewWriter(w) + + var bufZip []byte + var addr oid.Address + + empty := true + called := false + btoken := bearerToken(ctx) + addr.SetContainer(*containerID) + + errIter := resSearch.Iterate(func(id oid.ID) bool { + called = true + + if empty { + bufZip = make([]byte, 3<<20) // the same as for upload + } + empty = false + + addr.SetObject(id) + if err = h.zipObject(ctx, zipWriter, addr, btoken, bufZip); err != nil { + log.Error(logs.FailedToAddObjectToArchive, zap.String("oid", id.EncodeToString()), zap.Error(err)) + } + + return false + }) + if errIter != nil { + log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) + } else if !called { + log.Error(logs.ObjectsNotFound) + } + + if err = zipWriter.Close(); err != nil { + log.Error(logs.CloseZipWriter, zap.Error(err)) + } + }) +} + +func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error { + var prm pool.PrmObjectGet + prm.SetAddress(addr) + if btoken != nil { + prm.UseBearer(*btoken) + } + + resGet, err := h.pool.GetObject(ctx, prm) + if err != nil { + return fmt.Errorf("get FrostFS object: %v", err) + } + + objWriter, err := h.addObjectToZip(zipWriter, &resGet.Header) + if err != nil { + return fmt.Errorf("zip create header: %v", err) + } + + if _, err = io.CopyBuffer(objWriter, resGet.Payload, bufZip); err != nil { + return fmt.Errorf("copy object payload to zip file: %v", err) + } + + if err = resGet.Payload.Close(); err != nil { + return fmt.Errorf("object body close error: %w", err) + } + + if err = zipWriter.Flush(); err != nil { + return fmt.Errorf("flush zip writer: %v", err) + } + + return nil +} + +func getZipFilePath(obj *object.Object) string { + for _, attr := range obj.Attributes() { + if attr.Key() == object.AttributeFilePath { + return attr.Value() + } + } + + return "" +} diff --git a/uploader/filter.go b/internal/handler/filter.go similarity index 98% rename from uploader/filter.go rename to internal/handler/filter.go index 70d6eef..745718a 100644 --- a/uploader/filter.go +++ b/internal/handler/filter.go @@ -1,4 +1,4 @@ -package uploader +package handler import ( "bytes" diff --git a/uploader/filter_test.go b/internal/handler/filter_test.go similarity index 98% rename from uploader/filter_test.go rename to internal/handler/filter_test.go index 9d32b84..0322952 100644 --- a/uploader/filter_test.go +++ b/internal/handler/filter_test.go @@ -1,6 +1,6 @@ //go:build !integration -package uploader +package handler import ( "testing" diff --git a/internal/handler/handler.go b/internal/handler/handler.go new file mode 100644 index 0000000..d462280 --- /dev/null +++ b/internal/handler/handler.go @@ -0,0 +1,193 @@ +package handler + +import ( + "context" + "errors" + "io" + "net/url" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +type Handler struct { + log *zap.Logger + pool *pool.Pool + ownerID *user.ID + settings *Settings + containerResolver *resolver.ContainerResolver + tree *tree.Tree +} + +// Settings stores reloading parameters, so it has to provide atomic getters and setters. +type Settings struct { + defaultTimestamp atomic.Bool + zipCompression atomic.Bool +} + +func (s *Settings) DefaultTimestamp() bool { + return s.defaultTimestamp.Load() +} + +func (s *Settings) SetDefaultTimestamp(val bool) { + s.defaultTimestamp.Store(val) +} + +func (s *Settings) ZipCompression() bool { + return s.zipCompression.Load() +} + +func (s *Settings) SetZipCompression(val bool) { + s.zipCompression.Store(val) +} + +func New(params *utils.AppParams, settings *Settings, tree *tree.Tree) *Handler { + return &Handler{ + log: params.Logger, + pool: params.Pool, + ownerID: params.Owner, + settings: settings, + containerResolver: params.Resolver, + tree: tree, + } +} + +// getContainerID decode container id, if it's not a valid container id +// then trey to resolve name using provided resolver. +func (h *Handler) getContainerID(ctx context.Context, containerID string) (*cid.ID, error) { + cnrID := new(cid.ID) + err := cnrID.DecodeString(containerID) + if err != nil { + cnrID, err = h.containerResolver.Resolve(ctx, containerID) + } + return cnrID, err +} + +// byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that +// prepares request and object address to it. +func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { + var ( + idCnr, _ = c.UserValue("cid").(string) + idObj, _ = c.UserValue("oid").(string) + log = h.log.With(zap.String("cid", idCnr), zap.String("oid", idObj)) + ) + + ctx := utils.GetContextFromRequest(c) + + cnrID, err := h.getContainerID(ctx, idCnr) + if err != nil { + log.Error(logs.WrongContainerID, zap.Error(err)) + response.Error(c, "wrong container id", fasthttp.StatusBadRequest) + return + } + + objID := new(oid.ID) + if err = objID.DecodeString(idObj); err != nil { + log.Error(logs.WrongObjectID, zap.Error(err)) + response.Error(c, "wrong object id", fasthttp.StatusBadRequest) + return + } + + var addr oid.Address + addr.SetContainer(*cnrID) + addr.SetObject(*objID) + + f(ctx, *h.newRequest(c, log), addr) +} + +// byBucketname is a wrapper for function (e.g. request.headObject, request.receiveFile) that +// prepares request and object address to it. +func (h *Handler) byBucketname(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { + var ( + bucketname = req.UserValue("cid").(string) + key = req.UserValue("oid").(string) + log = h.log.With(zap.String("bucketname", bucketname), zap.String("key", key)) + ) + + ctx := utils.GetContextFromRequest(req) + + cnrID, err := h.getContainerID(ctx, bucketname) + if err != nil { + log.Error(logs.WrongContainerID, zap.Error(err)) + response.Error(req, "wrong container id", fasthttp.StatusBadRequest) + return + } + + foundOid, err := h.tree.GetLatestVersion(ctx, cnrID, key) + if err != nil { + log.Error(logs.ObjectWasntFound, zap.Error(err)) + response.Error(req, "object wasn't found", fasthttp.StatusNotFound) + return + } + if foundOid.DeleteMarker { + log.Error(logs.ObjectWasDeleted) + response.Error(req, "object deleted", fasthttp.StatusNotFound) + return + } + + var addr oid.Address + addr.SetContainer(*cnrID) + addr.SetObject(foundOid.OID) + + f(ctx, *h.newRequest(req, log), addr) +} + +// byAttribute is a wrapper similar to byAddress. +func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { + var ( + scid, _ = c.UserValue("cid").(string) + key, _ = url.QueryUnescape(c.UserValue("attr_key").(string)) + val, _ = url.QueryUnescape(c.UserValue("attr_val").(string)) + log = h.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) + ) + + ctx := utils.GetContextFromRequest(c) + + containerID, err := h.getContainerID(ctx, scid) + if err != nil { + log.Error(logs.WrongContainerID, zap.Error(err)) + response.Error(c, "wrong container id", fasthttp.StatusBadRequest) + return + } + + res, err := h.search(ctx, containerID, key, val, object.MatchStringEqual) + if err != nil { + log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + defer res.Close() + + buf := make([]oid.ID, 1) + + n, err := res.Read(buf) + if n == 0 { + if errors.Is(err, io.EOF) { + log.Error(logs.ObjectNotFound, zap.Error(err)) + response.Error(c, "object not found", fasthttp.StatusNotFound) + return + } + + log.Error(logs.ReadObjectListFailed, zap.Error(err)) + response.Error(c, "read object list failed: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + var addrObj oid.Address + addrObj.SetContainer(*containerID) + addrObj.SetObject(buf[0]) + + f(ctx, *h.newRequest(c, log), addrObj) +} diff --git a/downloader/head.go b/internal/handler/head.go similarity index 86% rename from downloader/head.go rename to internal/handler/head.go index 76dfd93..f7478f1 100644 --- a/downloader/head.go +++ b/internal/handler/head.go @@ -1,4 +1,4 @@ -package downloader +package handler import ( "context" @@ -25,7 +25,7 @@ const ( hdrContainerID = "X-Container-Id" ) -func headObject(ctx context.Context, req request, clnt *pool.Pool, objectAddress oid.Address) { +func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid.Address) { var start = time.Now() btoken := bearerToken(ctx) @@ -36,7 +36,7 @@ func headObject(ctx context.Context, req request, clnt *pool.Pool, objectAddress prm.UseBearer(*btoken) } - obj, err := clnt.HeadObject(ctx, prm) + obj, err := h.pool.HeadObject(ctx, prm) if err != nil { req.handleFrostFSErr(err, start) return @@ -81,7 +81,7 @@ func headObject(ctx context.Context, req request, clnt *pool.Pool, objectAddress prmRange.UseBearer(*btoken) } - resObj, err := clnt.ObjectRange(ctx, prmRange) + resObj, err := h.pool.ObjectRange(ctx, prmRange) if err != nil { return nil, err } @@ -104,19 +104,19 @@ func idsToResponse(resp *fasthttp.Response, obj *object.Object) { } // HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format. -func (d *Downloader) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { +func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { test, _ := c.UserValue("oid").(string) var id oid.ID err := id.DecodeString(test) if err != nil { - d.byBucketname(c, headObject) + h.byBucketname(c, h.headObject) } else { - d.byAddress(c, headObject) + h.byAddress(c, h.headObject) } } // HeadByAttribute handles attribute-based head requests. -func (d *Downloader) HeadByAttribute(c *fasthttp.RequestCtx) { - d.byAttribute(c, headObject) +func (h *Handler) HeadByAttribute(c *fasthttp.RequestCtx) { + h.byAttribute(c, h.headObject) } diff --git a/uploader/multipart.go b/internal/handler/multipart.go similarity index 92% rename from uploader/multipart.go rename to internal/handler/multipart.go index 135ee88..de9242f 100644 --- a/uploader/multipart.go +++ b/internal/handler/multipart.go @@ -1,10 +1,10 @@ -package uploader +package handler import ( "io" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/multipart" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader/multipart" "go.uber.org/zap" ) diff --git a/uploader/multipart/multipart.go b/internal/handler/multipart/multipart.go similarity index 100% rename from uploader/multipart/multipart.go rename to internal/handler/multipart/multipart.go diff --git a/uploader/multipart_test.go b/internal/handler/multipart_test.go similarity index 99% rename from uploader/multipart_test.go rename to internal/handler/multipart_test.go index d19cd5e..2c50a87 100644 --- a/uploader/multipart_test.go +++ b/internal/handler/multipart_test.go @@ -1,6 +1,6 @@ //go:build !integration -package uploader +package handler import ( "crypto/rand" diff --git a/internal/handler/reader.go b/internal/handler/reader.go new file mode 100644 index 0000000..76801f7 --- /dev/null +++ b/internal/handler/reader.go @@ -0,0 +1,141 @@ +package handler + +import ( + "bytes" + "context" + "io" + "net/http" + "path" + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +type readCloser struct { + io.Reader + io.Closer +} + +// initializes io.Reader with the limited size and detects Content-Type from it. +// Returns r's error directly. Also returns the processed data. +func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (string, []byte, error) { + if maxSize > sizeToDetectType { + maxSize = sizeToDetectType + } + + buf := make([]byte, maxSize) // maybe sync-pool the slice? + + r, err := rInit(maxSize) + if err != nil { + return "", nil, err + } + + n, err := r.Read(buf) + if err != nil && err != io.EOF { + return "", nil, err + } + + buf = buf[:n] + + return http.DetectContentType(buf), buf, err // to not lose io.EOF +} + +func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oid.Address) { + var ( + err error + dis = "inline" + start = time.Now() + filename string + ) + + var prm pool.PrmObjectGet + prm.SetAddress(objectAddress) + if btoken := bearerToken(ctx); btoken != nil { + prm.UseBearer(*btoken) + } + + rObj, err := h.pool.GetObject(ctx, prm) + if err != nil { + req.handleFrostFSErr(err, start) + return + } + + // we can't close reader in this function, so how to do it? + + if req.Request.URI().QueryArgs().GetBool("download") { + dis = "attachment" + } + + payloadSize := rObj.Header.PayloadSize() + + req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) + var contentType string + for _, attr := range rObj.Header.Attributes() { + key := attr.Key() + val := attr.Value() + if !isValidToken(key) || !isValidValue(val) { + continue + } + + key = utils.BackwardTransformIfSystem(key) + + req.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val) + switch key { + case object.AttributeFileName: + filename = val + case object.AttributeTimestamp: + value, err := strconv.ParseInt(val, 10, 64) + if err != nil { + req.log.Info(logs.CouldntParseCreationDate, + zap.String("key", key), + zap.String("val", val), + zap.Error(err)) + continue + } + req.Response.Header.Set(fasthttp.HeaderLastModified, + time.Unix(value, 0).UTC().Format(http.TimeFormat)) + case object.AttributeContentType: + contentType = val + } + } + + idsToResponse(&req.Response, &rObj.Header) + + if len(contentType) == 0 { + // determine the Content-Type from the payload head + var payloadHead []byte + + contentType, payloadHead, err = readContentType(payloadSize, func(uint64) (io.Reader, error) { + return rObj.Payload, nil + }) + if err != nil && err != io.EOF { + req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) + response.Error(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + // reset payload reader since a part of the data has been read + var headReader io.Reader = bytes.NewReader(payloadHead) + + if err != io.EOF { // otherwise, we've already read full payload + headReader = io.MultiReader(headReader, rObj.Payload) + } + + // note: we could do with io.Reader, but SetBodyStream below closes body stream + // if it implements io.Closer and that's useful for us. + rObj.Payload = readCloser{headReader, rObj.Payload} + } + req.SetContentType(contentType) + + req.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) + + req.Response.SetBodyStream(rObj.Payload, int(payloadSize)) +} diff --git a/downloader/reader_test.go b/internal/handler/reader_test.go similarity index 98% rename from downloader/reader_test.go rename to internal/handler/reader_test.go index 09c990a..73899ca 100644 --- a/downloader/reader_test.go +++ b/internal/handler/reader_test.go @@ -1,6 +1,6 @@ //go:build !integration -package downloader +package handler import ( "io" diff --git a/uploader/upload.go b/internal/handler/upload.go similarity index 73% rename from uploader/upload.go rename to internal/handler/upload.go index 2832043..8d4e681 100644 --- a/uploader/upload.go +++ b/internal/handler/upload.go @@ -1,4 +1,4 @@ -package uploader +package handler import ( "context" @@ -9,7 +9,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" @@ -17,9 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/valyala/fasthttp" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -28,55 +25,39 @@ const ( drainBufSize = 4096 ) -// Uploader is an upload request handler. -type Uploader struct { - log *zap.Logger - pool *pool.Pool - ownerID *user.ID - settings *Settings - containerResolver *resolver.ContainerResolver +type putResponse struct { + ObjectID string `json:"object_id"` + ContainerID string `json:"container_id"` } -// Settings stores reloading parameters, so it has to provide atomic getters and setters. -type Settings struct { - defaultTimestamp atomic.Bool -} - -func (s *Settings) DefaultTimestamp() bool { - return s.defaultTimestamp.Load() -} - -func (s *Settings) SetDefaultTimestamp(val bool) { - s.defaultTimestamp.Store(val) -} - -// New creates a new Uploader using specified logger, connection pool and -// other options. -func New(params *utils.AppParams, settings *Settings) *Uploader { - return &Uploader{ - log: params.Logger, - pool: params.Pool, - ownerID: params.Owner, - settings: settings, - containerResolver: params.Resolver, +func newPutResponse(addr oid.Address) *putResponse { + return &putResponse{ + ObjectID: addr.Object().EncodeToString(), + ContainerID: addr.Container().EncodeToString(), } } +func (pr *putResponse) encode(w io.Writer) error { + enc := json.NewEncoder(w) + enc.SetIndent("", "\t") + return enc.Encode(pr) +} + // Upload handles multipart upload request. -func (u *Uploader) Upload(req *fasthttp.RequestCtx) { +func (h *Handler) Upload(req *fasthttp.RequestCtx) { var ( file MultipartFile idObj oid.ID addr oid.Address scid, _ = req.UserValue("cid").(string) - log = u.log.With(zap.String("cid", scid)) + log = h.log.With(zap.String("cid", scid)) bodyStream = req.RequestBodyStream() drainBuf = make([]byte, drainBufSize) ) ctx := utils.GetContextFromRequest(req) - idCnr, err := utils.GetContainerID(ctx, scid, u.containerResolver) + idCnr, err := h.getContainerID(ctx, scid) if err != nil { log.Error(logs.WrongContainerID, zap.Error(err)) response.Error(req, "wrong container id", fasthttp.StatusBadRequest) @@ -97,12 +78,12 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { ) }() boundary := string(req.Request.Header.MultipartFormBoundary()) - if file, err = fetchMultipartFile(u.log, bodyStream, boundary); err != nil { + if file, err = fetchMultipartFile(h.log, bodyStream, boundary); err != nil { log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) response.Error(req, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } - filtered, err := filterHeaders(u.log, &req.Request.Header) + filtered, err := filterHeaders(h.log, &req.Request.Header) if err != nil { log.Error(logs.CouldNotProcessHeaders, zap.Error(err)) response.Error(req, err.Error(), fasthttp.StatusBadRequest) @@ -118,7 +99,7 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { } } - if err = utils.PrepareExpirationHeader(req, u.pool, filtered, now); err != nil { + if err = utils.PrepareExpirationHeader(req, h.pool, filtered, now); err != nil { log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) response.Error(req, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) return @@ -140,7 +121,7 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { attributes = append(attributes, *filename) } // sets Timestamp attribute if it wasn't set from header and enabled by settings - if _, ok := filtered[object.AttributeTimestamp]; !ok && u.settings.DefaultTimestamp() { + if _, ok := filtered[object.AttributeTimestamp]; !ok && h.settings.DefaultTimestamp() { timestamp := object.NewAttribute() timestamp.SetKey(object.AttributeTimestamp) timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) @@ -149,20 +130,20 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { obj := object.New() obj.SetContainerID(*idCnr) - obj.SetOwnerID(u.ownerID) + obj.SetOwnerID(h.ownerID) obj.SetAttributes(attributes...) var prm pool.PrmObjectPut prm.SetHeader(*obj) prm.SetPayload(file) - bt := u.fetchBearerToken(ctx) + bt := h.fetchBearerToken(ctx) if bt != nil { prm.UseBearer(*bt) } - if idObj, err = u.pool.PutObject(ctx, prm); err != nil { - u.handlePutFrostFSErr(req, err) + if idObj, err = h.pool.PutObject(ctx, prm); err != nil { + h.handlePutFrostFSErr(req, err) return } @@ -193,35 +174,17 @@ func (u *Uploader) Upload(req *fasthttp.RequestCtx) { req.Response.Header.SetContentType(jsonHeader) } -func (u *Uploader) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) { +func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) { statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err) logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) - u.log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) + h.log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) response.Error(r, msg, statusCode) } -func (u *Uploader) fetchBearerToken(ctx context.Context) *bearer.Token { +func (h *Handler) fetchBearerToken(ctx context.Context) *bearer.Token { if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil { return tkn } return nil } - -type putResponse struct { - ObjectID string `json:"object_id"` - ContainerID string `json:"container_id"` -} - -func newPutResponse(addr oid.Address) *putResponse { - return &putResponse{ - ObjectID: addr.Object().EncodeToString(), - ContainerID: addr.Container().EncodeToString(), - } -} - -func (pr *putResponse) encode(w io.Writer) error { - enc := json.NewEncoder(w) - enc.SetIndent("", "\t") - return enc.Encode(pr) -} diff --git a/internal/handler/utils.go b/internal/handler/utils.go new file mode 100644 index 0000000..b51400c --- /dev/null +++ b/internal/handler/utils.go @@ -0,0 +1,60 @@ +package handler + +import ( + "context" + "strings" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +type request struct { + *fasthttp.RequestCtx + log *zap.Logger +} + +func (r *request) handleFrostFSErr(err error, start time.Time) { + logFields := []zap.Field{ + zap.Stringer("elapsed", time.Since(start)), + zap.Error(err), + } + statusCode, msg, additionalFields := response.FormErrorResponse("could not receive object", err) + logFields = append(logFields, additionalFields...) + + r.log.Error(logs.CouldNotReceiveObject, logFields...) + response.Error(r.RequestCtx, msg, statusCode) +} + +func bearerToken(ctx context.Context) *bearer.Token { + if tkn, err := tokens.LoadBearerToken(ctx); err == nil { + return tkn + } + return nil +} + +func isValidToken(s string) bool { + for _, c := range s { + if c <= ' ' || c > 127 { + return false + } + if strings.ContainsRune("()<>@,;:\\\"/[]?={}", c) { + return false + } + } + return true +} + +func isValidValue(s string) bool { + for _, c := range s { + // HTTP specification allows for more technically, but we don't want to escape things. + if c < ' ' || c > 127 || c == '"' { + return false + } + } + return true +} diff --git a/tree/tree.go b/tree/tree.go index 84b6707..3a673b3 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -5,8 +5,8 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/api" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/api/layer" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) diff --git a/utils/util.go b/utils/util.go index e54c98d..a328769 100644 --- a/utils/util.go +++ b/utils/util.go @@ -4,23 +4,10 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" ) -// GetContainerID decode container id, if it's not a valid container id -// then trey to resolve name using provided resolver. -func GetContainerID(ctx context.Context, containerID string, resolver *resolver.ContainerResolver) (*cid.ID, error) { - cnrID := new(cid.ID) - err := cnrID.DecodeString(containerID) - if err != nil { - cnrID, err = resolver.Resolve(ctx, containerID) - } - return cnrID, err -} - type EpochDurations struct { CurrentEpoch uint64 MsPerBlock int64 From e26577e753c963258af7f5aa30241f404bbbc3b9 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Tue, 5 Sep 2023 18:17:22 +0300 Subject: [PATCH 417/548] [#74] Replace atomics with mutex for reloadable params Signed-off-by: Marina Biryukova --- cmd/http-gw/app.go | 39 ++++++++++++++++++++++++++++++++---- internal/handler/download.go | 2 +- internal/handler/handler.go | 34 ++++++++----------------------- internal/handler/upload.go | 2 +- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 40db433..e8ac917 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -50,7 +50,7 @@ type ( resolver *resolver.ContainerResolver metrics *gateMetrics services []*metrics.Service - settings *handler.Settings + settings *appSettings servers []Server } @@ -69,6 +69,13 @@ type ( mu sync.RWMutex enabled bool } + + // appSettings stores reloading parameters, so it has to provide getters and setters which use RWMutex. + appSettings struct { + mu sync.RWMutex + defaultTimestamp bool + zipCompression bool + } ) // WithLogger returns Option to set a specific logger. @@ -133,8 +140,32 @@ func newApp(ctx context.Context, opt ...Option) App { return a } +func (s *appSettings) DefaultTimestamp() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.defaultTimestamp +} + +func (s *appSettings) setDefaultTimestamp(val bool) { + s.mu.Lock() + s.defaultTimestamp = val + s.mu.Unlock() +} + +func (s *appSettings) ZipCompression() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.zipCompression +} + +func (s *appSettings) setZipCompression(val bool) { + s.mu.Lock() + s.zipCompression = val + s.mu.Unlock() +} + func (a *app) initAppSettings() { - a.settings = &handler.Settings{} + a.settings = &appSettings{} a.updateSettings() } @@ -415,8 +446,8 @@ func (a *app) configReload(ctx context.Context) { } func (a *app) updateSettings() { - a.settings.SetDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) - a.settings.SetZipCompression(a.cfg.GetBool(cfgZipCompression)) + a.settings.setDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) + a.settings.setZipCompression(a.cfg.GetBool(cfgZipCompression)) } func (a *app) startServices() { diff --git a/internal/handler/download.go b/internal/handler/download.go index 8ee76bf..696c57e 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -72,7 +72,7 @@ func (h *Handler) getContainer(ctx context.Context, cnrID cid.ID) (container.Con func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { method := zip.Store - if h.settings.ZipCompression() { + if h.config.ZipCompression() { method = zip.Deflate } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d462280..579a55f 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -5,7 +5,6 @@ import ( "errors" "io" "net/url" - "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" @@ -21,43 +20,26 @@ import ( "go.uber.org/zap" ) +type Config interface { + DefaultTimestamp() bool + ZipCompression() bool +} + type Handler struct { log *zap.Logger pool *pool.Pool ownerID *user.ID - settings *Settings + config Config containerResolver *resolver.ContainerResolver tree *tree.Tree } -// Settings stores reloading parameters, so it has to provide atomic getters and setters. -type Settings struct { - defaultTimestamp atomic.Bool - zipCompression atomic.Bool -} - -func (s *Settings) DefaultTimestamp() bool { - return s.defaultTimestamp.Load() -} - -func (s *Settings) SetDefaultTimestamp(val bool) { - s.defaultTimestamp.Store(val) -} - -func (s *Settings) ZipCompression() bool { - return s.zipCompression.Load() -} - -func (s *Settings) SetZipCompression(val bool) { - s.zipCompression.Store(val) -} - -func New(params *utils.AppParams, settings *Settings, tree *tree.Tree) *Handler { +func New(params *utils.AppParams, config Config, tree *tree.Tree) *Handler { return &Handler{ log: params.Logger, pool: params.Pool, ownerID: params.Owner, - settings: settings, + config: config, containerResolver: params.Resolver, tree: tree, } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 8d4e681..f5e0459 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -121,7 +121,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { attributes = append(attributes, *filename) } // sets Timestamp attribute if it wasn't set from header and enabled by settings - if _, ok := filtered[object.AttributeTimestamp]; !ok && h.settings.DefaultTimestamp() { + if _, ok := filtered[object.AttributeTimestamp]; !ok && h.config.DefaultTimestamp() { timestamp := object.NewAttribute() timestamp.SetKey(object.AttributeTimestamp) timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) From 84eb57475bfbccd303d68eddb1b0ecfc91771ca3 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Mon, 9 Oct 2023 09:41:17 +0300 Subject: [PATCH 418/548] [#85] Fix get latest version node Signed-off-by: Roman Loginov --- tree/tree.go | 39 ++++++++++++- tree/tree_test.go | 143 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 tree/tree_test.go diff --git a/tree/tree.go b/tree/tree.go index 3a673b3..a9135eb 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -73,6 +73,7 @@ type Meta interface { type NodeResponse interface { GetMeta() []Meta + GetTimestamp() uint64 } func newTreeNode(nodeInfo NodeResponse) (*treeNode, error) { @@ -135,7 +136,7 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s TreeID: versionTree, Path: path, Meta: meta, - LatestOnly: true, + LatestOnly: false, AllAttrs: false, } nodes, err := c.service.GetNodes(ctx, p) @@ -143,11 +144,43 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s return nil, err } - if len(nodes) == 0 { + latestNode, err := getLatestNode(nodes) + if err != nil { + return nil, err + } + + return newNodeVersion(latestNode) +} + +func getLatestNode(nodes []NodeResponse) (NodeResponse, error) { + var ( + maxCreationTime uint64 + targetIndexNode = -1 + ) + + for i, node := range nodes { + currentCreationTime := node.GetTimestamp() + if checkExistOID(node.GetMeta()) && currentCreationTime > maxCreationTime { + maxCreationTime = currentCreationTime + targetIndexNode = i + } + } + + if targetIndexNode == -1 { return nil, layer.ErrNodeNotFound } - return newNodeVersion(nodes[0]) + return nodes[targetIndexNode], nil +} + +func checkExistOID(meta []Meta) bool { + for _, kv := range meta { + if kv.GetKey() == "OID" { + return true + } + } + + return false } // pathFromName splits name by '/'. diff --git a/tree/tree_test.go b/tree/tree_test.go new file mode 100644 index 0000000..7cd2314 --- /dev/null +++ b/tree/tree_test.go @@ -0,0 +1,143 @@ +package tree + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +type nodeMeta struct { + key string + value []byte +} + +func (m nodeMeta) GetKey() string { + return m.key +} + +func (m nodeMeta) GetValue() []byte { + return m.value +} + +type nodeResponse struct { + meta []nodeMeta + timestamp uint64 +} + +func (n nodeResponse) GetTimestamp() uint64 { + return n.timestamp +} + +func (n nodeResponse) GetMeta() []Meta { + res := make([]Meta, len(n.meta)) + for i, value := range n.meta { + res[i] = value + } + return res +} + +func TestGetLatestNode(t *testing.T) { + for _, tc := range []struct { + name string + nodes []NodeResponse + exceptedOID string + error bool + }{ + { + name: "empty", + nodes: []NodeResponse{}, + error: true, + }, + { + name: "one node of the object version", + nodes: []NodeResponse{ + nodeResponse{ + timestamp: 1, + meta: []nodeMeta{ + { + key: oidKV, + value: []byte("oid1"), + }, + }, + }, + }, + exceptedOID: "oid1", + }, + { + name: "one node of the object version and one node of the secondary object", + nodes: []NodeResponse{ + nodeResponse{ + timestamp: 3, + meta: []nodeMeta{}, + }, + nodeResponse{ + timestamp: 1, + meta: []nodeMeta{ + { + key: oidKV, + value: []byte("oid1"), + }, + }, + }, + }, + exceptedOID: "oid1", + }, + { + name: "all nodes represent a secondary object", + nodes: []NodeResponse{ + nodeResponse{ + timestamp: 3, + meta: []nodeMeta{}, + }, + nodeResponse{ + timestamp: 5, + meta: []nodeMeta{}, + }, + }, + error: true, + }, + { + name: "several nodes of different types and with different timestamp", + nodes: []NodeResponse{ + nodeResponse{ + timestamp: 1, + meta: []nodeMeta{ + { + key: oidKV, + value: []byte("oid1"), + }, + }, + }, + nodeResponse{ + timestamp: 3, + meta: []nodeMeta{}, + }, + nodeResponse{ + timestamp: 4, + meta: []nodeMeta{ + { + key: oidKV, + value: []byte("oid2"), + }, + }, + }, + nodeResponse{ + timestamp: 6, + meta: []nodeMeta{}, + }, + }, + exceptedOID: "oid2", + }, + } { + t.Run(tc.name, func(t *testing.T) { + actualNode, err := getLatestNode(tc.nodes) + if tc.error { + require.Error(t, err) + return + } + + require.NoError(t, err) + require.Equal(t, tc.exceptedOID, string(actualNode.GetMeta()[0].GetValue())) + }) + } +} From e61b4867c9ff85c2c93d96475516eda95a63e99a Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 21 Aug 2023 16:30:19 +0300 Subject: [PATCH 419/548] [#70] Update SDK to support client cut Signed-off-by: Denis Kirillov --- go.mod | 2 +- go.sum | 4 ++-- internal/handler/download.go | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 80d794c..a58272e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230825064515-46a214d065f8 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc github.com/prometheus/client_golang v1.15.1 diff --git a/go.sum b/go.sum index f4c8c56..0b9b457 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6 h1:u6lzNotV6MEMNEG/XeS7g+FjPrrf+j4gnOHtvun2KJc= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802103237-363f153eafa6/go.mod h1:LI2GOj0pEx0jYTjB3QHja2PNhQFYL2pCm71RAFwDv0M= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230825064515-46a214d065f8 h1:0s2RkATjdtK/5fHjRGsIi8qMvc9IoeMZgMX5ddMwI+I= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230825064515-46a214d065f8/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/internal/handler/download.go b/internal/handler/download.go index 696c57e..5021b4a 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -64,8 +64,9 @@ func (h *Handler) search(ctx context.Context, cid *cid.ID, key, val string, op o } func (h *Handler) getContainer(ctx context.Context, cnrID cid.ID) (container.Container, error) { - var prm pool.PrmContainerGet - prm.SetContainerID(cnrID) + prm := pool.PrmContainerGet{ + ContainerID: cnrID, + } return h.pool.GetContainer(ctx, prm) } From 9b34413e17757af891cd56bd5ff74b99416ce3f2 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 21 Aug 2023 16:50:23 +0300 Subject: [PATCH 420/548] [#70] Support client cut Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + cmd/http-gw/app.go | 14 ++++++++++++++ cmd/http-gw/settings.go | 3 +++ config/config.env | 4 ++++ config/config.yaml | 5 +++++ docs/gate-configuration.md | 16 +++++++++++++++- internal/handler/handler.go | 1 + internal/handler/upload.go | 1 + 8 files changed, 44 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d323c89..d660f10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ This document outlines major changes between releases. - Support impersonate bearer token (#40, #45) - Tracing support (#20, #44, #60) - Object name resolving with tree service (#30) +- Add new `frostfs.client_cut` config param (#70) ### Changed - Update prometheus to v1.15.0 (#35) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index e8ac917..93a6b6c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -75,6 +75,7 @@ type ( mu sync.RWMutex defaultTimestamp bool zipCompression bool + clientCut bool } ) @@ -164,6 +165,18 @@ func (s *appSettings) setZipCompression(val bool) { s.mu.Unlock() } +func (s *appSettings) ClientCut() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.clientCut +} + +func (s *appSettings) setClientCut(val bool) { + s.mu.Lock() + s.clientCut = val + s.mu.Unlock() +} + func (a *app) initAppSettings() { a.settings = &appSettings{} @@ -448,6 +461,7 @@ func (a *app) configReload(ctx context.Context) { func (a *app) updateSettings() { a.settings.setDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) a.settings.setZipCompression(a.cfg.GetBool(cfgZipCompression)) + a.settings.setClientCut(a.cfg.GetBool(cfgClientCut)) } func (a *app) startServices() { diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 6708ad4..86718eb 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -96,6 +96,9 @@ const ( // Runtime. cfgSoftMemoryLimit = "runtime.soft_memory_limit" + // Enabling client side object preparing for PUT operations. + cfgClientCut = "frostfs.client_cut" + // Command line args. cmdHelp = "help" cmdVersion = "version" diff --git a/config/config.env b/config/config.env index 62920a2..58e6814 100644 --- a/config/config.env +++ b/config/config.env @@ -98,3 +98,7 @@ HTTP_GW_TRACING_ENDPOINT="localhost:4317" HTTP_GW_TRACING_EXPORTER="otlp_grpc" HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 + +# Parameters of requests to FrostFS +# This flag enables client side object preparing. +HTTP_GW_FROSTFS_CLIENT_CUT=false diff --git a/config/config.yaml b/config/config.yaml index d2804d6..3eae752 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -104,3 +104,8 @@ zip: runtime: soft_memory_limit: 1gb + +# Parameters of requests to FrostFS +frostfs: + # This flag enables client side object preparing. + client_cut: false diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 95e6c8e..25d068b 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -54,6 +54,7 @@ $ cat http.log | `prometheus` | [Prometheus configuration](#prometheus-section) | | `tracing` | [Tracing configuration](#tracing-section) | | `runtime` | [Runtime configuration](#runtime-section) | +| `frostfs` | [Frostfs configuration](#frostfs-section) | # General section @@ -268,4 +269,17 @@ runtime: | Parameter | Type | SIGHUP reload | Default value | Description | |---------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `soft_memory_limit` | `size` | yes | maxint64 | Soft memory limit for the runtime. Zero or no value stands for no limit. If `GOMEMLIMIT` environment variable is set, the value from the configuration file will be ignored. | \ No newline at end of file +| `soft_memory_limit` | `size` | yes | maxint64 | Soft memory limit for the runtime. Zero or no value stands for no limit. If `GOMEMLIMIT` environment variable is set, the value from the configuration file will be ignored. | + +# `frostfs` section + +Contains parameters of requests to FrostFS. + +```yaml +frostfs: + client_cut: false +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|--------------|--------|---------------|---------------|-------------------------------------------------| +| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. | diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 579a55f..2bb4347 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -23,6 +23,7 @@ import ( type Config interface { DefaultTimestamp() bool ZipCompression() bool + ClientCut() bool } type Handler struct { diff --git a/internal/handler/upload.go b/internal/handler/upload.go index f5e0459..a8c4365 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -136,6 +136,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { var prm pool.PrmObjectPut prm.SetHeader(*obj) prm.SetPayload(file) + prm.SetClientCut(h.config.ClientCut()) bt := h.fetchBearerToken(ctx) if bt != nil { From 8bc246f8f91582f32f8b8889f89f935bf418de17 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 25 Aug 2023 14:53:59 +0300 Subject: [PATCH 421/548] [#70] Support configuring buffer size for put Signed-off-by: Denis Kirillov --- CHANGELOG.md | 2 +- cmd/http-gw/app.go | 22 ++++++++++++++++++---- cmd/http-gw/settings.go | 7 +++++++ config/config.env | 2 ++ config/config.yaml | 2 ++ docs/gate-configuration.md | 8 +++++--- internal/handler/handler.go | 1 + internal/handler/upload.go | 1 + 8 files changed, 37 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d660f10..10d42c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ This document outlines major changes between releases. - Support impersonate bearer token (#40, #45) - Tracing support (#20, #44, #60) - Object name resolving with tree service (#30) -- Add new `frostfs.client_cut` config param (#70) +- Add new `frostfs.client_cut` and `frostfs.buffer_max_size_for_put` config params (#70) ### Changed - Update prometheus to v1.15.0 (#35) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 93a6b6c..81bfb14 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -72,10 +72,11 @@ type ( // appSettings stores reloading parameters, so it has to provide getters and setters which use RWMutex. appSettings struct { - mu sync.RWMutex - defaultTimestamp bool - zipCompression bool - clientCut bool + mu sync.RWMutex + defaultTimestamp bool + zipCompression bool + clientCut bool + bufferMaxSizeForPut uint64 } ) @@ -177,6 +178,18 @@ func (s *appSettings) setClientCut(val bool) { s.mu.Unlock() } +func (s *appSettings) BufferMaxSizeForPut() uint64 { + s.mu.RLock() + defer s.mu.RUnlock() + return s.bufferMaxSizeForPut +} + +func (s *appSettings) setBufferMaxSizeForPut(val uint64) { + s.mu.Lock() + s.bufferMaxSizeForPut = val + s.mu.Unlock() +} + func (a *app) initAppSettings() { a.settings = &appSettings{} @@ -462,6 +475,7 @@ func (a *app) updateSettings() { a.settings.setDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) a.settings.setZipCompression(a.cfg.GetBool(cfgZipCompression)) a.settings.setClientCut(a.cfg.GetBool(cfgClientCut)) + a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut)) } func (a *app) startServices() { diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 86718eb..b097aa3 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -39,6 +39,8 @@ const ( defaultSoftMemoryLimit = math.MaxInt64 + defaultBufferMaxSizeForPut = 1024 * 1024 // 1mb + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" @@ -98,6 +100,8 @@ const ( // Enabling client side object preparing for PUT operations. cfgClientCut = "frostfs.client_cut" + // Sets max buffer size for read payload in put operations. + cfgBufferMaxSizeForPut = "frostfs.buffer_max_size_for_put" // Command line args. cmdHelp = "help" @@ -160,6 +164,9 @@ func settings() *viper.Viper { // pool: v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) + // frostfs: + v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut) + // web-server: v.SetDefault(cfgWebReadBufferSize, 4096) v.SetDefault(cfgWebWriteBufferSize, 4096) diff --git a/config/config.env b/config/config.env index 58e6814..06e71f8 100644 --- a/config/config.env +++ b/config/config.env @@ -102,3 +102,5 @@ HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 # Parameters of requests to FrostFS # This flag enables client side object preparing. HTTP_GW_FROSTFS_CLIENT_CUT=false +# Sets max buffer size for read payload in put operations. +HTTP_GW_FROSTFS_BUFFER_MAX_SIZE_FOR_PUT=1048576 diff --git a/config/config.yaml b/config/config.yaml index 3eae752..15afeec 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -109,3 +109,5 @@ runtime: frostfs: # This flag enables client side object preparing. client_cut: false + # Sets max buffer size for read payload in put operations. + buffer_max_size_for_put: 1048576 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 25d068b..852908b 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -278,8 +278,10 @@ Contains parameters of requests to FrostFS. ```yaml frostfs: client_cut: false + buffer_max_size_for_put: 1048576 # 1mb ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|--------------|--------|---------------|---------------|-------------------------------------------------| -| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------------------|----------|---------------|---------------|----------------------------------------------------------| +| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. | +| `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. | diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2bb4347..61e2bc1 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -24,6 +24,7 @@ type Config interface { DefaultTimestamp() bool ZipCompression() bool ClientCut() bool + BufferMaxSizeForPut() uint64 } type Handler struct { diff --git a/internal/handler/upload.go b/internal/handler/upload.go index a8c4365..b95896f 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -137,6 +137,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { prm.SetHeader(*obj) prm.SetPayload(file) prm.SetClientCut(h.config.ClientCut()) + prm.SetBufferMaxSize(h.config.BufferMaxSizeForPut()) bt := h.fetchBearerToken(ctx) if bt != nil { From 9a5a2239bd0ef00e32f68dcb4d478293cb67dedf Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 4 Oct 2023 14:50:37 +0300 Subject: [PATCH 422/548] [#70] Support bucket/container caching Mainly it was added because we need to know if TZ hashing is disabled or not for container Signed-off-by: Denis Kirillov --- CHANGELOG.md | 6 ++- cmd/http-gw/app.go | 2 + cmd/http-gw/settings.go | 46 ++++++++++++++++++ config/config.env | 5 ++ config/config.yaml | 7 +++ docs/gate-configuration.md | 29 ++++++++++++ go.mod | 1 + go.sum | 2 + internal/cache/buckets.go | 68 +++++++++++++++++++++++++++ internal/data/bucket.go | 12 +++++ internal/handler/download.go | 34 ++------------ internal/handler/handler.go | 90 +++++++++++++++++++++++++++++++----- internal/handler/head.go | 2 +- internal/handler/upload.go | 10 ++-- internal/handler/utils.go | 11 +++++ internal/logs/logs.go | 4 +- utils/params.go | 2 + 17 files changed, 283 insertions(+), 48 deletions(-) create mode 100644 internal/cache/buckets.go create mode 100644 internal/data/bucket.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 10d42c0..4618d35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,11 @@ This document outlines major changes between releases. - Support impersonate bearer token (#40, #45) - Tracing support (#20, #44, #60) - Object name resolving with tree service (#30) -- Add new `frostfs.client_cut` and `frostfs.buffer_max_size_for_put` config params (#70) +- Support client side object cut (#70) + - Add `frostfs.client_cut` config param + - Add `frostfs.buffer_max_size_for_put` config param + - Add bucket/container caching + - Disable homomorphic hash for PUT if it's disabled in container itself ### Changed - Update prometheus to v1.15.0 (#35) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 81bfb14..3878277 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -11,6 +11,7 @@ import ( "syscall" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" @@ -567,6 +568,7 @@ func (a *app) AppParams() *utils.AppParams { Pool: a.pool, Owner: a.owner, Resolver: a.resolver, + Cache: cache.NewBucketCache(getCacheOptions(a.cfg, a.log)), } } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index b097aa3..cb309b7 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" @@ -103,6 +104,10 @@ const ( // Sets max buffer size for read payload in put operations. cfgBufferMaxSizeForPut = "frostfs.buffer_max_size_for_put" + // Caching. + cfgBucketsCacheLifetime = "cache.buckets.lifetime" + cfgBucketsCacheSize = "cache.buckets.size" + // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -541,3 +546,44 @@ func fetchSoftMemoryLimit(cfg *viper.Viper) int64 { return int64(softMemoryLimit) } + +func getCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { + cacheCfg := cache.DefaultBucketConfig(l) + + cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgBucketsCacheLifetime, cacheCfg.Lifetime) + cacheCfg.Size = fetchCacheSize(v, l, cfgBucketsCacheSize, cacheCfg.Size) + + return cacheCfg +} + +func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration { + if v.IsSet(cfgEntry) { + lifetime := v.GetDuration(cfgEntry) + if lifetime <= 0 { + l.Error("invalid lifetime, using default value (in seconds)", + zap.String("parameter", cfgEntry), + zap.Duration("value in config", lifetime), + zap.Duration("default", defaultValue)) + } else { + return lifetime + } + } + + return defaultValue +} + +func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue int) int { + if v.IsSet(cfgEntry) { + size := v.GetInt(cfgEntry) + if size <= 0 { + l.Error("invalid cache size, using default value", + zap.String("parameter", cfgEntry), + zap.Int("value in config", size), + zap.Int("default", defaultValue)) + } else { + return size + } + } + + return defaultValue +} diff --git a/config/config.env b/config/config.env index 06e71f8..739cb96 100644 --- a/config/config.env +++ b/config/config.env @@ -104,3 +104,8 @@ HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 HTTP_GW_FROSTFS_CLIENT_CUT=false # Sets max buffer size for read payload in put operations. HTTP_GW_FROSTFS_BUFFER_MAX_SIZE_FOR_PUT=1048576 + +# Caching +# Cache which contains mapping of bucket name to bucket info +HTTP_GW_CACHE_BUCKETS_LIFETIME=1m +HTTP_GW_CACHE_BUCKETS_SIZE=1000 diff --git a/config/config.yaml b/config/config.yaml index 15afeec..2cd20b5 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -111,3 +111,10 @@ frostfs: client_cut: false # Sets max buffer size for read payload in put operations. buffer_max_size_for_put: 1048576 + +# Caching +cache: + # Cache which contains mapping of bucket name to bucket info + buckets: + lifetime: 1m + size: 1000 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 852908b..65fe618 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -55,6 +55,7 @@ $ cat http.log | `tracing` | [Tracing configuration](#tracing-section) | | `runtime` | [Runtime configuration](#runtime-section) | | `frostfs` | [Frostfs configuration](#frostfs-section) | +| `cache` | [Cache configuration](#cache-section) | # General section @@ -285,3 +286,31 @@ frostfs: |---------------------------|----------|---------------|---------------|----------------------------------------------------------| | `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. | | `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. | + + +### `cache` section + +```yaml +cache: + buckets: + lifetime: 1m + size: 1000 + +``` + +| Parameter | Type | Default value | Description | +|-----------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------------------------| +| `buckets` | [Cache config](#cache-subsection) | `lifetime: 60s`
`size: 1000` | Cache which contains mapping of bucket name to bucket info. | + + +#### `cache` subsection + +```yaml +lifetime: 1m +size: 1000 +``` + +| Parameter | Type | Default value | Description | +|------------|------------|------------------|-------------------------------| +| `lifetime` | `duration` | depends on cache | Lifetime of entries in cache. | +| `size` | `int` | depends on cache | LRU cache size. | diff --git a/go.mod b/go.mod index a58272e..5f9b1b0 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230825064515-46a214d065f8 + github.com/bluele/gcache v0.0.2 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc github.com/prometheus/client_golang v1.15.1 diff --git a/go.sum b/go.sum index 0b9b457..dedb570 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,8 @@ github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngE github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= diff --git a/internal/cache/buckets.go b/internal/cache/buckets.go new file mode 100644 index 0000000..abeda6a --- /dev/null +++ b/internal/cache/buckets.go @@ -0,0 +1,68 @@ +package cache + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "github.com/bluele/gcache" + "go.uber.org/zap" +) + +// BucketCache contains cache with objects and the lifetime of cache entries. +type BucketCache struct { + cache gcache.Cache + logger *zap.Logger +} + +// Config stores expiration params for cache. +type Config struct { + Size int + Lifetime time.Duration + Logger *zap.Logger +} + +const ( + // DefaultBucketCacheSize is a default maximum number of entries in cache. + DefaultBucketCacheSize = 1e3 + // DefaultBucketCacheLifetime is a default lifetime of entries in cache. + DefaultBucketCacheLifetime = time.Minute +) + +// DefaultBucketConfig returns new default cache expiration values. +func DefaultBucketConfig(logger *zap.Logger) *Config { + return &Config{ + Size: DefaultBucketCacheSize, + Lifetime: DefaultBucketCacheLifetime, + Logger: logger, + } +} + +// NewBucketCache creates an object of BucketCache. +func NewBucketCache(config *Config) *BucketCache { + gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + return &BucketCache{cache: gc, logger: config.Logger} +} + +// Get returns a cached object. +func (o *BucketCache) Get(key string) *data.BucketInfo { + entry, err := o.cache.Get(key) + if err != nil { + return nil + } + + result, ok := entry.(*data.BucketInfo) + if !ok { + o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result))) + return nil + } + + return result +} + +// Put puts an object to cache. +func (o *BucketCache) Put(bkt *data.BucketInfo) error { + return o.cache.Set(bkt.Name, bkt) +} diff --git a/internal/data/bucket.go b/internal/data/bucket.go new file mode 100644 index 0000000..d99ca49 --- /dev/null +++ b/internal/data/bucket.go @@ -0,0 +1,12 @@ +package data + +import ( + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" +) + +type BucketInfo struct { + Name string // container name from system attribute + Zone string // container zone from system attribute + CID cid.ID + HomomorphicHashDisabled bool +} diff --git a/internal/handler/download.go b/internal/handler/download.go index 5021b4a..06a247a 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -14,8 +14,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -30,7 +28,7 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { var id oid.ID err := id.DecodeString(test) if err != nil { - h.byBucketname(c, h.receiveFile) + h.byObjectName(c, h.receiveFile) } else { h.byAddress(c, h.receiveFile) } @@ -63,14 +61,6 @@ func (h *Handler) search(ctx context.Context, cid *cid.ID, key, val string, op o return h.pool.SearchObjects(ctx, prm) } -func (h *Handler) getContainer(ctx context.Context, cnrID cid.ID) (container.Container, error) { - prm := pool.PrmContainerGet{ - ContainerID: cnrID, - } - - return h.pool.GetContainer(ctx, prm) -} - func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { method := zip.Store if h.config.ZipCompression() { @@ -97,27 +87,13 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { ctx := utils.GetContextFromRequest(c) - containerID, err := h.getContainerID(ctx, scid) + bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(c, "wrong container id", fasthttp.StatusBadRequest) + logAndSendBucketError(c, log, err) return } - // check if container exists here to be able to return 404 error, - // otherwise we get this error only in object iteration step - // and client get 200 OK. - if _, err = h.getContainer(ctx, *containerID); err != nil { - log.Error(logs.CouldNotCheckContainerExistence, zap.Error(err)) - if client.IsErrContainerNotFound(err) { - response.Error(c, "Not Found", fasthttp.StatusNotFound) - return - } - response.Error(c, "could not check container existence: "+err.Error(), fasthttp.StatusBadRequest) - return - } - - resSearch, err := h.search(ctx, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) + resSearch, err := h.search(ctx, &bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) @@ -139,7 +115,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { empty := true called := false btoken := bearerToken(ctx) - addr.SetContainer(*containerID) + addr.SetContainer(bktInfo.CID) errIter := resSearch.Iterate(func(id oid.ID) bool { called = true diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 61e2bc1..54602c2 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -3,14 +3,20 @@ package handler import ( "context" "errors" + "fmt" "io" "net/url" + "strings" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -34,6 +40,7 @@ type Handler struct { config Config containerResolver *resolver.ContainerResolver tree *tree.Tree + cache *cache.BucketCache } func New(params *utils.AppParams, config Config, tree *tree.Tree) *Handler { @@ -44,6 +51,7 @@ func New(params *utils.AppParams, config Config, tree *tree.Tree) *Handler { config: config, containerResolver: params.Resolver, tree: tree, + cache: params.Cache, } } @@ -69,10 +77,9 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ ctx := utils.GetContextFromRequest(c) - cnrID, err := h.getContainerID(ctx, idCnr) + bktInfo, err := h.getBucketInfo(ctx, idCnr, log) if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(c, "wrong container id", fasthttp.StatusBadRequest) + logAndSendBucketError(c, log, err) return } @@ -84,15 +91,15 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ } var addr oid.Address - addr.SetContainer(*cnrID) + addr.SetContainer(bktInfo.CID) addr.SetObject(*objID) f(ctx, *h.newRequest(c, log), addr) } -// byBucketname is a wrapper for function (e.g. request.headObject, request.receiveFile) that +// byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byBucketname(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { +func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { var ( bucketname = req.UserValue("cid").(string) key = req.UserValue("oid").(string) @@ -101,14 +108,13 @@ func (h *Handler) byBucketname(req *fasthttp.RequestCtx, f func(context.Context, ctx := utils.GetContextFromRequest(req) - cnrID, err := h.getContainerID(ctx, bucketname) + bktInfo, err := h.getBucketInfo(ctx, bucketname, log) if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(req, "wrong container id", fasthttp.StatusBadRequest) + logAndSendBucketError(req, log, err) return } - foundOid, err := h.tree.GetLatestVersion(ctx, cnrID, key) + foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, key) if err != nil { log.Error(logs.ObjectWasntFound, zap.Error(err)) response.Error(req, "object wasn't found", fasthttp.StatusNotFound) @@ -121,7 +127,7 @@ func (h *Handler) byBucketname(req *fasthttp.RequestCtx, f func(context.Context, } var addr oid.Address - addr.SetContainer(*cnrID) + addr.SetContainer(bktInfo.CID) addr.SetObject(foundOid.OID) f(ctx, *h.newRequest(req, log), addr) @@ -175,3 +181,65 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re f(ctx, *h.newRequest(c, log), addrObj) } + +// resolveContainer decode container id, if it's not a valid container id +// then trey to resolve name using provided resolver. +func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) { + cnrID := new(cid.ID) + err := cnrID.DecodeString(containerID) + if err != nil { + cnrID, err = h.containerResolver.Resolve(ctx, containerID) + if err != nil && strings.Contains(err.Error(), "not found") { + err = fmt.Errorf("%w: %s", &apistatus.ContainerNotFound{}, err.Error()) + + } + } + return cnrID, err +} + +func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log *zap.Logger) (*data.BucketInfo, error) { + if bktInfo := h.cache.Get(containerName); bktInfo != nil { + return bktInfo, nil + } + + cnrID, err := h.resolveContainer(ctx, containerName) + if err != nil { + return nil, err + } + + bktInfo, err := h.readContainer(ctx, *cnrID) + if err != nil { + return nil, err + } + + if err = h.cache.Put(bktInfo); err != nil { + log.Warn(logs.CouldntPutBucketIntoCache, + zap.String("bucket name", bktInfo.Name), + zap.Stringer("bucket cid", bktInfo.CID), + zap.Error(err)) + } + + return bktInfo, nil +} + +func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) { + prm := pool.PrmContainerGet{ContainerID: cnrID} + res, err := h.pool.GetContainer(ctx, prm) + if err != nil { + return nil, fmt.Errorf("get frostfs container '%s': %w", cnrID.String(), err) + } + + bktInfo := &data.BucketInfo{ + CID: cnrID, + Name: cnrID.EncodeToString(), + } + + if domain := container.ReadDomain(res); domain.Name() != "" { + bktInfo.Name = domain.Name() + bktInfo.Zone = domain.Zone() + } + + bktInfo.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(res) + + return bktInfo, err +} diff --git a/internal/handler/head.go b/internal/handler/head.go index f7478f1..9418567 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -110,7 +110,7 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { err := id.DecodeString(test) if err != nil { - h.byBucketname(c, h.headObject) + h.byObjectName(c, h.headObject) } else { h.byAddress(c, h.headObject) } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index b95896f..935b51b 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -57,10 +57,9 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { ctx := utils.GetContextFromRequest(req) - idCnr, err := h.getContainerID(ctx, scid) + bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(req, "wrong container id", fasthttp.StatusBadRequest) + logAndSendBucketError(req, log, err) return } @@ -129,7 +128,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { } obj := object.New() - obj.SetContainerID(*idCnr) + obj.SetContainerID(bktInfo.CID) obj.SetOwnerID(h.ownerID) obj.SetAttributes(attributes...) @@ -138,6 +137,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { prm.SetPayload(file) prm.SetClientCut(h.config.ClientCut()) prm.SetBufferMaxSize(h.config.BufferMaxSizeForPut()) + prm.WithoutHomomorphicHash(bktInfo.HomomorphicHashDisabled) bt := h.fetchBearerToken(ctx) if bt != nil { @@ -150,7 +150,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { } addr.SetObject(idObj) - addr.SetContainer(*idCnr) + addr.SetContainer(bktInfo.CID) // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(req); err != nil { diff --git a/internal/handler/utils.go b/internal/handler/utils.go index b51400c..a5a53ed 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -58,3 +59,13 @@ func isValidValue(s string) bool { } return true } + +func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { + log.Error(logs.CouldntGetBucket, zap.Error(err)) + + if client.IsErrContainerNotFound(err) { + response.Error(c, "Not Found", fasthttp.StatusNotFound) + return + } + response.Error(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index ebb3c24..79ddce5 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -11,7 +11,6 @@ const ( CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go ObjectNotFound = "object not found" // Error in ../../downloader/download.go ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go - CouldNotCheckContainerExistence = "could not check container existence" // Error in ../../downloader/download.go FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go @@ -68,4 +67,7 @@ const ( FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../settings.go FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../settings.go AddedStoragePeer = "added storage peer" // Info in ../../settings.go + CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go + CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go + InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go ) diff --git a/utils/params.go b/utils/params.go index a6fe59b..f27ff71 100644 --- a/utils/params.go +++ b/utils/params.go @@ -1,6 +1,7 @@ package utils import ( + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -12,4 +13,5 @@ type AppParams struct { Pool *pool.Pool Owner *user.ID Resolver *resolver.ContainerResolver + Cache *cache.BucketCache } From 49d6a2756291452ad212bcaf4f96f165741ed41f Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 4 Oct 2023 15:39:44 +0300 Subject: [PATCH 423/548] [#70] Adjust status codes Signed-off-by: Denis Kirillov --- internal/handler/handler.go | 30 +++++++++++------------------- internal/logs/logs.go | 2 +- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 54602c2..fa3c364 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -55,17 +55,6 @@ func New(params *utils.AppParams, config Config, tree *tree.Tree) *Handler { } } -// getContainerID decode container id, if it's not a valid container id -// then trey to resolve name using provided resolver. -func (h *Handler) getContainerID(ctx context.Context, containerID string) (*cid.ID, error) { - cnrID := new(cid.ID) - err := cnrID.DecodeString(containerID) - if err != nil { - cnrID, err = h.containerResolver.Resolve(ctx, containerID) - } - return cnrID, err -} - // byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { @@ -116,7 +105,12 @@ func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, key) if err != nil { - log.Error(logs.ObjectWasntFound, zap.Error(err)) + if errors.Is(err, tree.ErrNodeAccessDenied) { + response.Error(req, "Access Denied", fasthttp.StatusForbidden) + return + } + log.Error(logs.GetLatestObjectVersion, zap.Error(err)) + response.Error(req, "object wasn't found", fasthttp.StatusNotFound) return } @@ -144,14 +138,13 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re ctx := utils.GetContextFromRequest(c) - containerID, err := h.getContainerID(ctx, scid) + bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(c, "wrong container id", fasthttp.StatusBadRequest) + logAndSendBucketError(c, log, err) return } - res, err := h.search(ctx, containerID, key, val, object.MatchStringEqual) + res, err := h.search(ctx, &bktInfo.CID, key, val, object.MatchStringEqual) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) @@ -176,7 +169,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re } var addrObj oid.Address - addrObj.SetContainer(*containerID) + addrObj.SetContainer(bktInfo.CID) addrObj.SetObject(buf[0]) f(ctx, *h.newRequest(c, log), addrObj) @@ -190,8 +183,7 @@ func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*ci if err != nil { cnrID, err = h.containerResolver.Resolve(ctx, containerID) if err != nil && strings.Contains(err.Error(), "not found") { - err = fmt.Errorf("%w: %s", &apistatus.ContainerNotFound{}, err.Error()) - + err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error()) } } return cnrID, err diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 79ddce5..0534ebc 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -6,7 +6,7 @@ const ( CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go WrongContainerID = "wrong container id" // Error in ../../downloader/download.go and uploader/upload.go WrongObjectID = "wrong object id" // Error in ../../downloader/download.go - ObjectWasntFound = "object wasn't found" // Error in ../../downloader/download.go + GetLatestObjectVersion = "get latest object version" // Error in ../../downloader/download.go ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go ObjectNotFound = "object not found" // Error in ../../downloader/download.go From 1ced82a71475ae8c6dac9f78185294af1985b05e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 6 Oct 2023 16:14:09 +0300 Subject: [PATCH 424/548] [#70] Fix log messages (move to constants) Signed-off-by: Denis Kirillov --- cmd/http-gw/settings.go | 4 ++-- internal/logs/logs.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index cb309b7..7e53158 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -560,7 +560,7 @@ func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultV if v.IsSet(cfgEntry) { lifetime := v.GetDuration(cfgEntry) if lifetime <= 0 { - l.Error("invalid lifetime, using default value (in seconds)", + l.Error(logs.InvalidLifetimeUsingDefaultValue, zap.String("parameter", cfgEntry), zap.Duration("value in config", lifetime), zap.Duration("default", defaultValue)) @@ -576,7 +576,7 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue if v.IsSet(cfgEntry) { size := v.GetInt(cfgEntry) if size <= 0 { - l.Error("invalid cache size, using default value", + l.Error(logs.InvalidCacheSizeUsingDefaultValue, zap.String("parameter", cfgEntry), zap.Int("value in config", size), zap.Int("default", defaultValue)) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 0534ebc..9d464c3 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -70,4 +70,7 @@ const ( CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go + InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go + InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go + ) From 7fa973b26123532a12a3c2b08bec045038c602cf Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 9 Nov 2023 11:11:30 +0300 Subject: [PATCH 425/548] [#89] Add support zapjournald logger configuration Signed-off-by: Roman Loginov --- CHANGELOG.md | 1 + cmd/http-gw/main.go | 2 +- cmd/http-gw/settings.go | 57 ++++++++++++++++++++++++++++++++------ config/config.yaml | 1 + docs/gate-configuration.md | 9 +++--- go.mod | 2 ++ go.sum | 4 +++ 7 files changed, 63 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4618d35..fd19460 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ This document outlines major changes between releases. - Add `frostfs.buffer_max_size_for_put` config param - Add bucket/container caching - Disable homomorphic hash for PUT if it's disabled in container itself +- Add new `logger.destination` config param (#89) ### Changed - Update prometheus to v1.15.0 (#35) diff --git a/cmd/http-gw/main.go b/cmd/http-gw/main.go index 5762675..ea9fbd7 100644 --- a/cmd/http-gw/main.go +++ b/cmd/http-gw/main.go @@ -9,7 +9,7 @@ import ( func main() { globalContext, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) v := settings() - logger, atomicLevel := newLogger(v) + logger, atomicLevel := pickLogger(v) application := newApp(globalContext, WithLogger(logger, atomicLevel), WithConfig(v)) go application.Serve() diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 7e53158..6e633ba 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -19,15 +19,22 @@ import ( grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" + "git.frostfs.info/TrueCloudLab/zapjournald" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/pflag" "github.com/spf13/viper" + "github.com/ssgreg/journald" "github.com/valyala/fasthttp" "go.uber.org/zap" "go.uber.org/zap/zapcore" "google.golang.org/grpc" ) +const ( + destinationStdout = "stdout" + destinationJournald = "journald" +) + const ( defaultRebalanceTimer = 60 * time.Second defaultRequestTimeout = 15 * time.Second @@ -74,7 +81,8 @@ const ( cfgPoolErrorThreshold = "pool_error_threshold" // Logger. - cfgLoggerLevel = "logger.level" + cfgLoggerLevel = "logger.level" + cfgLoggerDestination = "logger.destination" // Wallet. cfgWalletPassphrase = "wallet.passphrase" @@ -165,6 +173,7 @@ func settings() *viper.Viper { // logger: v.SetDefault(cfgLoggerLevel, "debug") + v.SetDefault(cfgLoggerDestination, "stdout") // pool: v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) @@ -349,7 +358,25 @@ func mergeConfig(v *viper.Viper, fileName string) error { return v.MergeConfig(cfgFile) } -// newLogger constructs a zap.Logger instance for current application. +func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { + lvl, err := getLogLevel(v) + if err != nil { + panic(err) + } + + dest := v.GetString(cfgLoggerDestination) + + switch dest { + case destinationStdout: + return newStdoutLogger(lvl) + case destinationJournald: + return newJournaldLogger(lvl) + default: + panic(fmt.Sprintf("wrong destination for logger: %s", dest)) + } +} + +// newStdoutLogger constructs a zap.Logger instance for current application. // Panics on failure. // // Logger is built from zap's production logging configuration with: @@ -360,12 +387,7 @@ func mergeConfig(v *viper.Viper, fileName string) error { // Logger records a stack trace for all messages at or above fatal level. // // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. -func newLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { - lvl, err := getLogLevel(v) - if err != nil { - panic(err) - } - +func newStdoutLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { c := zap.NewProductionConfig() c.Level = zap.NewAtomicLevelAt(lvl) c.Encoding = "console" @@ -381,6 +403,25 @@ func newLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { return l, c.Level } +func newJournaldLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { + c := zap.NewProductionConfig() + c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + c.Level = zap.NewAtomicLevelAt(lvl) + + encoder := zapcore.NewConsoleEncoder(c.EncoderConfig) + + core := zapjournald.NewCore(zap.NewAtomicLevelAt(lvl), encoder, &journald.Journal{}, zapjournald.SyslogFields) + coreWithContext := core.With([]zapcore.Field{ + zapjournald.SyslogFacility(zapjournald.LogDaemon), + zapjournald.SyslogIdentifier(), + zapjournald.SyslogPid(), + }) + + l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) + + return l, c.Level +} + func getLogLevel(v *viper.Viper) (zapcore.Level, error) { var lvl zapcore.Level lvlStr := v.GetString(cfgLoggerLevel) diff --git a/config/config.yaml b/config/config.yaml index 2cd20b5..6ab9994 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -16,6 +16,7 @@ tracing: logger: level: debug # Log level. + destination: stdout server: - address: 0.0.0.0:8080 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 65fe618..1b51848 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -160,12 +160,13 @@ server: ```yaml logger: level: debug + destination: stdout ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-----------|----------|---------------|---------------|----------------------------------------------------------------------------------------------------| -| `level` | `string` | yes | `debug` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | - +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------|----------|---------------|---------------|----------------------------------------------------------------------------------------------------| +| `level` | `string` | yes | `debug` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | +| `destination` | `string` | no | `stdout` | Destination for logger: `stdout` or `journald` | # `web` section diff --git a/go.mod b/go.mod index 5f9b1b0..98abcb7 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230825064515-46a214d065f8 + git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20231018083019-2b6d84de9a3d github.com/bluele/gcache v0.0.2 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc @@ -13,6 +14,7 @@ require ( github.com/prometheus/client_model v0.3.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 + github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.8.3 github.com/testcontainers/testcontainers-go v0.13.0 github.com/valyala/fasthttp v1.34.0 diff --git a/go.sum b/go.sum index dedb570..5d46822 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,8 @@ git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9m git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= +git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20231018083019-2b6d84de9a3d h1:Z9UuI+jxzPtwQZUMmATdTuA8/8l2jzBY1rVh/gwBDsw= +git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20231018083019-2b6d84de9a3d/go.mod h1:rQFJJdEOV7KbbMtQYR2lNfiZk+ONRDJSbMCTWxKt8Fw= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -873,6 +875,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= +github.com/ssgreg/journald v1.0.0 h1:0YmTDPJXxcWDPba12qNMdO6TxvfkFSYpFIJ31CwmLcU= +github.com/ssgreg/journald v1.0.0/go.mod h1:RUckwmTM8ghGWPslq2+ZBZzbb9/2KgjzYZ4JEP+oRt0= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From dc8d0d4ab380b66586522a449aac902bc2102e1a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 22 Nov 2023 11:56:59 +0300 Subject: [PATCH 426/548] [#95] Add dirty version check Signed-off-by: Alex Vanin --- .forgejo/workflows/builds.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.forgejo/workflows/builds.yml b/.forgejo/workflows/builds.yml index aac6857..97ac86b 100644 --- a/.forgejo/workflows/builds.yml +++ b/.forgejo/workflows/builds.yml @@ -18,3 +18,6 @@ jobs: - name: Build binary run: make + + - name: Check dirty suffix + run: if [[ $(make version) == *"dirty"* ]]; then echo "Version has dirty suffix" && exit 1; fi From a375af7d98373a5d7177a04755bc87d95f1b77ef Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Tue, 28 Nov 2023 11:29:08 +0300 Subject: [PATCH 427/548] [#91] Add support namespaces Signed-off-by: Roman Loginov --- CHANGELOG.md | 3 +- cmd/http-gw/app.go | 66 ++++++++++++++++++++++++++--- cmd/http-gw/integration_test.go | 38 ++++++++++++++++- cmd/http-gw/settings.go | 10 +++++ config/config.env | 5 +++ config/config.yaml | 4 ++ docs/gate-configuration.md | 49 ++++++++++++++------- go.mod | 2 +- internal/cache/buckets.go | 10 +++-- internal/handler/handler.go | 9 +++- internal/handler/middleware/util.go | 26 ++++++++++++ resolver/resolver.go | 51 ++++++++++++++++++---- 12 files changed, 236 insertions(+), 37 deletions(-) create mode 100644 internal/handler/middleware/util.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fd19460..e2dd2c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ This document outlines major changes between releases. - Add `frostfs.buffer_max_size_for_put` config param - Add bucket/container caching - Disable homomorphic hash for PUT if it's disabled in container itself -- Add new `logger.destination` config param (#89) +- Add new `logger.destination` config param (#89) +- Add support namespaces (#91) ### Changed - Update prometheus to v1.15.0 (#35) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 3878277..1ad1f20 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -7,13 +7,16 @@ import ( "os" "os/signal" "runtime/debug" + "strings" "sync" "syscall" "time" + v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" @@ -34,6 +37,7 @@ import ( "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.uber.org/zap" + "golang.org/x/exp/slices" ) type ( @@ -78,6 +82,8 @@ type ( zipCompression bool clientCut bool bufferMaxSizeForPut uint64 + namespaceHeader string + defaultNamespaces []string } ) @@ -209,6 +215,7 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) { resolveCfg := &resolver.Config{ FrostFS: resolver.NewFrostFSResolver(a.pool), RPCAddress: a.cfg.GetString(cfgRPCEndpoint), + Settings: a.settings, } order := a.cfg.GetStringSlice(cfgResolveOrder) @@ -477,6 +484,8 @@ func (a *app) updateSettings() { a.settings.setZipCompression(a.cfg.GetBool(cfgZipCompression)) a.settings.setClientCut(a.cfg.GetBool(cfgClientCut)) a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut)) + a.settings.setNamespaceHeader(a.cfg.GetString(cfgResolveNamespaceHeader)) + a.settings.setDefaultNamespaces(a.cfg.GetStringSlice(cfgResolveDefaultNamespaces)) } func (a *app) startServices() { @@ -510,15 +519,15 @@ func (a *app) configureRouter(handler *handler.Handler) { response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.logger(a.tokenizer(a.tracer(handler.Upload)))) + r.POST("/upload/{cid}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.Upload))))) a.log.Info(logs.AddedPathUploadCid) - r.GET("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(handler.DownloadByAddressOrBucketName)))) - r.HEAD("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(handler.HeadByAddressOrBucketName)))) + r.GET("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadByAddressOrBucketName))))) + r.HEAD("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.HeadByAddressOrBucketName))))) a.log.Info(logs.AddedPathGetCidOid) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(handler.DownloadByAttribute)))) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(handler.HeadByAttribute)))) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadByAttribute))))) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.HeadByAttribute))))) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.logger(a.tokenizer(a.tracer(handler.DownloadZipped)))) + r.GET("/zip/{cid}/{prefix:*}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadZipped))))) a.log.Info(logs.AddedPathZipCidPrefix) a.webServer.Handler = r.Handler @@ -562,6 +571,18 @@ func (a *app) tracer(h fasthttp.RequestHandler) fasthttp.RequestHandler { } } +func (a *app) reqNamespace(h fasthttp.RequestHandler) fasthttp.RequestHandler { + return func(req *fasthttp.RequestCtx) { + appCtx := utils.GetContextFromRequest(req) + + nsBytes := req.Request.Header.Peek(a.settings.NamespaceHeader()) + appCtx = middleware.SetNamespace(appCtx, string(nsBytes)) + + utils.SetContextToRequest(appCtx, req) + h(req) + } +} + func (a *app) AppParams() *utils.AppParams { return &utils.AppParams{ Logger: a.log, @@ -669,3 +690,36 @@ func (a *app) setRuntimeParameters() { zap.Int64("old_value", previous)) } } + +func (s *appSettings) NamespaceHeader() string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.namespaceHeader +} + +func (s *appSettings) setNamespaceHeader(nsHeader string) { + s.mu.Lock() + s.namespaceHeader = nsHeader + s.mu.Unlock() +} + +func (s *appSettings) FormContainerZone(ns string) (zone string, isDefault bool) { + s.mu.RLock() + namespaces := s.defaultNamespaces + s.mu.RUnlock() + if slices.Contains(namespaces, ns) { + return v2container.SysAttributeZoneDefault, true + } + + return ns + ".ns", false +} + +func (s *appSettings) setDefaultNamespaces(namespaces []string) { + for i := range namespaces { // to be set namespaces in env variable as `HTTP_GW_RESOLVE_BUCKET_DEFAULT_NAMESPACES="" "root"` + namespaces[i] = strings.Trim(namespaces[i], "\"") + } + + s.mu.Lock() + s.defaultNamespaces = namespaces + s.mu.Unlock() +} diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 76a8325..f76c3ce 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -29,6 +29,7 @@ import ( "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" + "go.uber.org/zap/zapcore" ) type putResponse struct { @@ -68,6 +69,7 @@ func TestIntegration(t *testing.T) { t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, ownerID, CID, version) }) t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID, version) }) t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, ownerID, CID, version) }) + t.Run("test namespaces "+version, func(t *testing.T) { checkNamespaces(ctx, t, clientPool, ownerID, CID, version) }) cancel() server.Wait() @@ -81,7 +83,7 @@ func runServer() (App, context.CancelFunc) { cancelCtx, cancel := context.WithCancel(context.Background()) v := getDefaultConfig() - l, lvl := newLogger(v) + l, lvl := newStdoutLogger(zapcore.DebugLevel) application := newApp(cancelCtx, WithConfig(v), WithLogger(l, lvl)) go application.Serve() @@ -338,6 +340,40 @@ func checkZip(t *testing.T, data []byte, length int64, names, contents []string) } } +func checkNamespaces(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { + content := "content of file" + attributes := map[string]string{ + "some-attr": "some-get-value", + } + + id := putObject(ctx, t, clientPool, ownerID, CID, content, attributes) + + req, err := http.NewRequest(http.MethodGet, testHost+"/get/"+testContainerName+"/"+id.String(), nil) + require.NoError(t, err) + req.Header.Set(defaultNamespaceHeader, "") + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + checkGetResponse(t, resp, content, attributes) + + req, err = http.NewRequest(http.MethodGet, testHost+"/get/"+testContainerName+"/"+id.String(), nil) + require.NoError(t, err) + req.Header.Set(defaultNamespaceHeader, "root") + + resp, err = http.DefaultClient.Do(req) + require.NoError(t, err) + checkGetResponse(t, resp, content, attributes) + + req, err = http.NewRequest(http.MethodGet, testHost+"/get/"+testContainerName+"/"+id.String(), nil) + require.NoError(t, err) + req.Header.Set(defaultNamespaceHeader, "root2") + + resp, err = http.DefaultClient.Do(req) + require.NoError(t, err) + require.Equal(t, http.StatusNotFound, resp.StatusCode) + +} + func createDockerContainer(ctx context.Context, t *testing.T, image string) testcontainers.Container { req := testcontainers.ContainerRequest{ Image: image, diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 6e633ba..24e4f37 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -49,6 +49,8 @@ const ( defaultBufferMaxSizeForPut = 1024 * 1024 // 1mb + defaultNamespaceHeader = "X-Frostfs-Namespace" + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" @@ -116,6 +118,10 @@ const ( cfgBucketsCacheLifetime = "cache.buckets.lifetime" cfgBucketsCacheSize = "cache.buckets.size" + // Bucket resolving options. + cfgResolveNamespaceHeader = "resolve_bucket.namespace_header" + cfgResolveDefaultNamespaces = "resolve_bucket.default_namespaces" + // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -199,6 +205,10 @@ func settings() *viper.Viper { v.SetDefault(cfgPprofAddress, "localhost:8083") v.SetDefault(cfgPrometheusAddress, "localhost:8084") + // resolve bucket + v.SetDefault(cfgResolveNamespaceHeader, defaultNamespaceHeader) + v.SetDefault(cfgResolveDefaultNamespaces, []string{"", "root"}) + // Binding flags if err := v.BindPFlag(cfgPprofEnabled, flags.Lookup(cmdPprof)); err != nil { panic(err) diff --git a/config/config.env b/config/config.env index 739cb96..be42af9 100644 --- a/config/config.env +++ b/config/config.env @@ -109,3 +109,8 @@ HTTP_GW_FROSTFS_BUFFER_MAX_SIZE_FOR_PUT=1048576 # Cache which contains mapping of bucket name to bucket info HTTP_GW_CACHE_BUCKETS_LIFETIME=1m HTTP_GW_CACHE_BUCKETS_SIZE=1000 + +# Header to determine zone to resolve bucket name +HTTP_GW_RESOLVE_BUCKET_NAMESPACE_HEADER=X-Frostfs-Namespace +# Namespaces that should be handled as default +HTTP_GW_RESOLVE_BUCKET_DEFAULT_NAMESPACES="" "root" diff --git a/config/config.yaml b/config/config.yaml index 6ab9994..020b0dd 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -119,3 +119,7 @@ cache: buckets: lifetime: 1m size: 1000 + +resolve_bucket: + namespace_header: X-Frostfs-Namespace + default_namespaces: [ "", "root" ] \ No newline at end of file diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 1b51848..fe4b50f 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -40,22 +40,23 @@ $ cat http.log # Structure -| Section | Description | -|-----------------|-------------------------------------------------------| -| no section | [General parameters](#general-section) | -| `wallet` | [Wallet configuration](#wallet-section) | -| `peers` | [Nodes configuration](#peers-section) | -| `logger` | [Logger configuration](#logger-section) | -| `web` | [Web configuration](#web-section) | -| `server` | [Server configuration](#server-section) | -| `upload-header` | [Upload header configuration](#upload-header-section) | -| `zip` | [ZIP configuration](#zip-section) | -| `pprof` | [Pprof configuration](#pprof-section) | -| `prometheus` | [Prometheus configuration](#prometheus-section) | -| `tracing` | [Tracing configuration](#tracing-section) | -| `runtime` | [Runtime configuration](#runtime-section) | -| `frostfs` | [Frostfs configuration](#frostfs-section) | -| `cache` | [Cache configuration](#cache-section) | +| Section | Description | +|------------------|----------------------------------------------------------------| +| no section | [General parameters](#general-section) | +| `wallet` | [Wallet configuration](#wallet-section) | +| `peers` | [Nodes configuration](#peers-section) | +| `logger` | [Logger configuration](#logger-section) | +| `web` | [Web configuration](#web-section) | +| `server` | [Server configuration](#server-section) | +| `upload-header` | [Upload header configuration](#upload-header-section) | +| `zip` | [ZIP configuration](#zip-section) | +| `pprof` | [Pprof configuration](#pprof-section) | +| `prometheus` | [Prometheus configuration](#prometheus-section) | +| `tracing` | [Tracing configuration](#tracing-section) | +| `runtime` | [Runtime configuration](#runtime-section) | +| `frostfs` | [Frostfs configuration](#frostfs-section) | +| `cache` | [Cache configuration](#cache-section) | +| `resolve_bucket` | [Bucket name resolving configuration](#resolve_bucket-section) | # General section @@ -315,3 +316,19 @@ size: 1000 |------------|------------|------------------|-------------------------------| | `lifetime` | `duration` | depends on cache | Lifetime of entries in cache. | | `size` | `int` | depends on cache | LRU cache size. | + + +# `resolve_bucket` section + +Bucket name resolving parameters from and to container ID. + +```yaml +resolve_bucket: + namespace_header: X-Frostfs-Namespace + default_namespaces: [ "", "root" ] +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|----------------------|------------|---------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------| +| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. | +| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | \ No newline at end of file diff --git a/go.mod b/go.mod index 98abcb7..c398358 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.24.0 + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc google.golang.org/grpc v1.55.0 ) @@ -102,7 +103,6 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.9.0 // indirect - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.8.0 // indirect diff --git a/internal/cache/buckets.go b/internal/cache/buckets.go index abeda6a..f8e6d88 100644 --- a/internal/cache/buckets.go +++ b/internal/cache/buckets.go @@ -46,8 +46,8 @@ func NewBucketCache(config *Config) *BucketCache { } // Get returns a cached object. -func (o *BucketCache) Get(key string) *data.BucketInfo { - entry, err := o.cache.Get(key) +func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo { + entry, err := o.cache.Get(formKey(ns, bktName)) if err != nil { return nil } @@ -64,5 +64,9 @@ func (o *BucketCache) Get(key string) *data.BucketInfo { // Put puts an object to cache. func (o *BucketCache) Put(bkt *data.BucketInfo) error { - return o.cache.Set(bkt.Name, bkt) + return o.cache.Set(formKey(bkt.Zone, bkt.Name), bkt) +} + +func formKey(ns, name string) string { + return name + "." + ns } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index fa3c364..757b5be 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" @@ -31,6 +32,7 @@ type Config interface { ZipCompression() bool ClientCut() bool BufferMaxSizeForPut() uint64 + NamespaceHeader() string } type Handler struct { @@ -190,7 +192,12 @@ func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*ci } func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log *zap.Logger) (*data.BucketInfo, error) { - if bktInfo := h.cache.Get(containerName); bktInfo != nil { + ns, err := middleware.GetNamespace(ctx) + if err != nil { + return nil, err + } + + if bktInfo := h.cache.Get(ns, containerName); bktInfo != nil { return bktInfo, nil } diff --git a/internal/handler/middleware/util.go b/internal/handler/middleware/util.go new file mode 100644 index 0000000..284513a --- /dev/null +++ b/internal/handler/middleware/util.go @@ -0,0 +1,26 @@ +package middleware + +import ( + "context" + "fmt" +) + +// keyWrapper is wrapper for context keys. +type keyWrapper string + +const nsKey = keyWrapper("namespace") + +// GetNamespace extract namespace from context. +func GetNamespace(ctx context.Context) (string, error) { + ns, ok := ctx.Value(nsKey).(string) + if !ok { + return "", fmt.Errorf("couldn't get namespace from context") + } + + return ns, nil +} + +// SetNamespace sets namespace in the context. +func SetNamespace(ctx context.Context, ns string) context.Context { + return context.WithValue(ctx, nsKey, ns) +} diff --git a/resolver/resolver.go b/resolver/resolver.go index e6707e2..e7615d4 100644 --- a/resolver/resolver.go +++ b/resolver/resolver.go @@ -6,6 +6,7 @@ import ( "fmt" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns" @@ -28,9 +29,14 @@ type FrostFS interface { SystemDNS(context.Context) (string, error) } +type Settings interface { + FormContainerZone(ns string) (zone string, isDefault bool) +} + type Config struct { FrostFS FrostFS RPCAddress string + Settings Settings } type ContainerResolver struct { @@ -135,29 +141,43 @@ func (r *ContainerResolver) equals(resolverNames []string) bool { func newResolver(name string, cfg *Config) (*Resolver, error) { switch name { case DNSResolver: - return NewDNSResolver(cfg.FrostFS) + return NewDNSResolver(cfg.FrostFS, cfg.Settings) case NNSResolver: - return NewNNSResolver(cfg.RPCAddress) + return NewNNSResolver(cfg.RPCAddress, cfg.Settings) default: return nil, fmt.Errorf("unknown resolver: %s", name) } } -func NewDNSResolver(frostFS FrostFS) (*Resolver, error) { +func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) { if frostFS == nil { return nil, fmt.Errorf("pool must not be nil for DNS resolver") } + if settings == nil { + return nil, fmt.Errorf("resolver settings must not be nil for DNS resolver") + } var dns ns.DNS resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) { - domain, err := frostFS.SystemDNS(ctx) + var err error + + namespace, err := middleware.GetNamespace(ctx) if err != nil { - return nil, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err) + return nil, err } - domain = name + "." + domain + zone, isDefault := settings.FormContainerZone(namespace) + if isDefault { + zone, err = frostFS.SystemDNS(ctx) + if err != nil { + return nil, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err) + } + } + + domain := name + "." + zone cnrID, err := dns.ResolveContainerName(domain) + if err != nil { return nil, fmt.Errorf("couldn't resolve container '%s' as '%s': %w", name, domain, err) } @@ -170,17 +190,32 @@ func NewDNSResolver(frostFS FrostFS) (*Resolver, error) { }, nil } -func NewNNSResolver(rpcAddress string) (*Resolver, error) { +func NewNNSResolver(rpcAddress string, settings Settings) (*Resolver, error) { + if rpcAddress == "" { + return nil, fmt.Errorf("rpc address must not be empty for NNS resolver") + } + if settings == nil { + return nil, fmt.Errorf("resolver settings must not be nil for NNS resolver") + } + var nns ns.NNS if err := nns.Dial(rpcAddress); err != nil { return nil, fmt.Errorf("could not dial nns: %w", err) } - resolveFunc := func(_ context.Context, name string) (*cid.ID, error) { + resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) { var d container.Domain d.SetName(name) + namespace, err := middleware.GetNamespace(ctx) + if err != nil { + return nil, err + } + + zone, _ := settings.FormContainerZone(namespace) + d.SetZone(zone) + cnrID, err := nns.ResolveContainerDomain(d) if err != nil { return nil, fmt.Errorf("couldn't resolve container '%s': %w", name, err) From 2e28b2ac85a905a92f064f58f7aa843fddd4b5f8 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 7 Dec 2023 16:28:12 +0300 Subject: [PATCH 428/548] Release v0.28.0 Signed-off-by: Alex Vanin --- CHANGELOG.md | 31 +++++++++++++++++++++++++------ VERSION | 2 +- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2dd2c9..23878ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,13 +5,8 @@ This document outlines major changes between releases. ## [Unreleased] ### Fixed -- `grpc` schemas in tree configuration (#62) ### Added -- Support dump metrics descriptions (#29) -- Support impersonate bearer token (#40, #45) -- Tracing support (#20, #44, #60) -- Object name resolving with tree service (#30) - Support client side object cut (#70) - Add `frostfs.client_cut` config param - Add `frostfs.buffer_max_size_for_put` config param @@ -20,11 +15,34 @@ This document outlines major changes between releases. - Add new `logger.destination` config param (#89) - Add support namespaces (#91) +### Changed + +### Removed + +## [0.28.0] - Academy of Sciences - 2023-12-07 + +### Fixed +- `grpc` schemas in tree configuration (#62) +- `GetSubTree` failures (#67) +- Debian packaging (#69, #90) +- Get latest version of tree node (#85) + +### Added +- Support dump metrics descriptions (#29) +- Support impersonate bearer token (#40, #45) +- Tracing support (#20, #44, #60) +- Object name resolving with tree service (#30) +- Metrics for current endpoint status (#77) +- Soft memory limit with `runtime.soft_memory_limit` (#72) +- Add selection of the node of the latest version of the object (#85) + ### Changed - Update prometheus to v1.15.0 (#35) - Update go version to 1.19 (#50) - Finish rebranding (#2) - Use gate key to form object owner (#66) +- Move log messages to constants (#36) +- Uploader and downloader refactor (#73) ### Removed - Drop `tree.service` param (now endpoints from `peers` section are used) (#59) @@ -68,4 +86,5 @@ This project is a fork of [NeoFS HTTP Gateway](https://github.com/nspcc-dev/neof To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs-http-gw/blob/master/CHANGELOG.md. [0.27.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/72734ab4...v0.27.0 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.27.0...master +[0.28.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.27.0...v0.28.0 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.0...master diff --git a/VERSION b/VERSION index 0a8bf80..31950da 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.27.0 +v0.28.0 From 0ef3e18ee1bf2df75de4657477e20672d8a5e105 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 7 Nov 2023 11:00:38 +0300 Subject: [PATCH 429/548] [#92] Set tree request id Signed-off-by: Denis Kirillov --- cmd/http-gw/app.go | 3 +++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 1ad1f20..aabb22b 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -7,6 +7,7 @@ import ( "os" "os/signal" "runtime/debug" + "strconv" "strings" "sync" "syscall" @@ -566,6 +567,8 @@ func (a *app) tracer(h fasthttp.RequestHandler) fasthttp.RequestHandler { span.End() }() + appCtx = treepool.SetRequestID(appCtx, strconv.FormatUint(req.ID(), 10)) + utils.SetContextToRequest(appCtx, req) h(req) } diff --git a/go.mod b/go.mod index c398358..6791920 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230825064515-46a214d065f8 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20231018083019-2b6d84de9a3d github.com/bluele/gcache v0.0.2 github.com/fasthttp/router v1.4.1 diff --git a/go.sum b/go.sum index 5d46822..d32d051 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230825064515-46a214d065f8 h1:0s2RkATjdtK/5fHjRGsIi8qMvc9IoeMZgMX5ddMwI+I= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230825064515-46a214d065f8/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939 h1:jZEepi9yWmqrWgLRQcHQu4YPJaudmd7d2AEhpmM3m4U= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 627294bf704c5e3ca2e0ac0b23ff0a8e4b92a935 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 7 Nov 2023 11:07:27 +0300 Subject: [PATCH 430/548] [#92] Support configuring max tree request attempts Signed-off-by: Denis Kirillov --- cmd/http-gw/settings.go | 5 +++++ config/config.env | 4 ++++ config/config.yaml | 3 +++ docs/gate-configuration.md | 10 ++++++---- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 24e4f37..789dd2f 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -113,6 +113,9 @@ const ( cfgClientCut = "frostfs.client_cut" // Sets max buffer size for read payload in put operations. cfgBufferMaxSizeForPut = "frostfs.buffer_max_size_for_put" + // Configuration of parameters of requests to FrostFS. + // Sets max attempt to make successful tree request. + cfgTreePoolMaxAttempts = "frostfs.tree_pool_max_attempts" // Caching. cfgBucketsCacheLifetime = "cache.buckets.lifetime" @@ -527,6 +530,8 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool. prm.SetLogger(logger) prmTree.SetLogger(logger) + prmTree.SetMaxRequestAttempts(cfg.GetInt(cfgTreePoolMaxAttempts)) + var apiGRPCDialOpts []grpc.DialOption var treeGRPCDialOpts []grpc.DialOption if cfg.GetBool(cfgTracingEnabled) { diff --git a/config/config.env b/config/config.env index be42af9..12f1ba4 100644 --- a/config/config.env +++ b/config/config.env @@ -114,3 +114,7 @@ HTTP_GW_CACHE_BUCKETS_SIZE=1000 HTTP_GW_RESOLVE_BUCKET_NAMESPACE_HEADER=X-Frostfs-Namespace # Namespaces that should be handled as default HTTP_GW_RESOLVE_BUCKET_DEFAULT_NAMESPACES="" "root" + +# Max attempt to make successful tree request. +# default value is 0 that means the number of attempts equals to number of nodes in pool. +HTTP_GW_FROSTFS_TREE_POOL_MAX_ATTEMPTS=0 diff --git a/config/config.yaml b/config/config.yaml index 020b0dd..7ea2748 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -112,6 +112,9 @@ frostfs: client_cut: false # Sets max buffer size for read payload in put operations. buffer_max_size_for_put: 1048576 + # Max attempt to make successful tree request. + # default value is 0 that means the number of attempts equals to number of nodes in pool. + tree_pool_max_attempts: 0 # Caching cache: diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index fe4b50f..bf792b7 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -282,12 +282,14 @@ Contains parameters of requests to FrostFS. frostfs: client_cut: false buffer_max_size_for_put: 1048576 # 1mb + tree_pool_max_attempts: 0 ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|---------------------------|----------|---------------|---------------|----------------------------------------------------------| -| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. | -| `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------| +| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. | +| `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. | +| `tree_pool_max_attempts` | `uint32` | no | `0` | Sets max attempt to make successful tree request. Value 0 means the number of attempts equals to number of nodes in pool. | ### `cache` section From 5ae75eb9d8ccf16ad3d4bbb6da7047c2589536a0 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 21 Nov 2023 16:37:50 +0300 Subject: [PATCH 431/548] [#94] Update api-go to fix stable marshal of empty structs Newer version of api-go does not ignore non-nil empty structures in protobuf messages, so compatibility with previous version is preserved. Signed-off-by: Alex Vanin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6791920..422ded9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.20 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20231018083019-2b6d84de9a3d diff --git a/go.sum b/go.sum index d32d051..aec5152 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 h1:v6JqBD/VzZx3QSxbaXnUwnnJ1KEYheU4LzLGr3IhsAE= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4 h1:wjLfZ3WCt7qNGsQv+Jl0TXnmtg0uVk/jToKPFTBc/jo= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= From 2c95250f725322115021958232e1d27ef3244cd6 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Wed, 27 Dec 2023 13:33:09 +0300 Subject: [PATCH 432/548] [#99] Fix possibility of panic during SIGHUP Signed-off-by: Marina Biryukova --- CHANGELOG.md | 1 + internal/logs/logs.go | 1 + metrics/service.go | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23878ca..79ef8d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This document outlines major changes between releases. ## [Unreleased] ### Fixed +- Fix possibility of panic during SIGHUP (#99) ### Added - Support client side object cut (#70) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 9d464c3..c75e91f 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -20,6 +20,7 @@ const ( ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go ShuttingDownService = "shutting down service" // Info in ../../metrics/service.go CantShutDownService = "can't shut down service" // Panic in ../../metrics/service.go + CantGracefullyShutDownService = "can't gracefully shut down service, force stop" // Error in ../../metrics/service.go IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go diff --git a/metrics/service.go b/metrics/service.go index c025f06..dea5ac0 100644 --- a/metrics/service.go +++ b/metrics/service.go @@ -40,6 +40,9 @@ func (ms *Service) ShutDown(ctx context.Context) { ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr)) err := ms.Shutdown(ctx) if err != nil { - ms.log.Panic(logs.CantShutDownService) + ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err)) + if err = ms.Close(); err != nil { + ms.log.Panic(logs.CantShutDownService, zap.Error(err)) + } } } From 4049255eed5339c6007dba0509a72dd048c1691a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 24 Jan 2024 15:06:26 +0300 Subject: [PATCH 433/548] [#102] Port release v0.28.1 changelog Signed-off-by: Alex Vanin --- CHANGELOG.md | 11 ++++++++++- VERSION | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79ef8d8..c8022ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ This document outlines major changes between releases. ## [Unreleased] +## [0.28.1] - 2024-01-24 + +### Added +- Tree pool traversal limit (#92) + +### Update from 0.28.0 +See new `frostfs.tree_pool_max_attempts` config parameter. + ### Fixed - Fix possibility of panic during SIGHUP (#99) @@ -88,4 +96,5 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.27.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/72734ab4...v0.27.0 [0.28.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.27.0...v0.28.0 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.0...master +[0.28.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.0...v0.28.1 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.1...master diff --git a/VERSION b/VERSION index 31950da..244df55 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.28.0 +v0.28.1 From ce4ec032f93004e48c9c3fe9604ad4845f6af730 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 26 Jan 2024 11:29:32 +0300 Subject: [PATCH 434/548] [#103] .forgejo: Update dco-go to v3 Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/dco.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 3d38c4b..a9c3697 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -15,6 +15,6 @@ jobs: go-version: '1.21' - name: Run commit format checker - uses: https://git.frostfs.info/TrueCloudLab/dco-go@v1 + uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 with: from: adb95642d From c0389576492d415cf346b32b613d75adc47c92d5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 26 Jan 2024 11:37:29 +0300 Subject: [PATCH 435/548] [#103] .forgejo: Check only PR commits in dco-go Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/dco.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index a9c3697..eb23ec5 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -17,4 +17,4 @@ jobs: - name: Run commit format checker uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 with: - from: adb95642d + from: 'origin/${{ github.event.pull_request.base.ref }}' From 54709163613f179fb8298f31617f8a7903b2874d Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Mon, 29 Jan 2024 16:02:22 +0300 Subject: [PATCH 436/548] [#104] journald update We want to have less useless fields in logs Signed-off-by: Pavel Pogodaev --- cmd/http-gw/settings.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 789dd2f..045a118 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -421,7 +421,7 @@ func newJournaldLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder c.Level = zap.NewAtomicLevelAt(lvl) - encoder := zapcore.NewConsoleEncoder(c.EncoderConfig) + encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields) core := zapjournald.NewCore(zap.NewAtomicLevelAt(lvl), encoder, &journald.Journal{}, zapjournald.SyslogFields) coreWithContext := core.With([]zapcore.Field{ diff --git a/go.mod b/go.mod index 422ded9..9388eae 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939 - git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20231018083019-2b6d84de9a3d + git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc diff --git a/go.sum b/go.sum index aec5152..1bd67bc 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,8 @@ git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9m git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= -git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20231018083019-2b6d84de9a3d h1:Z9UuI+jxzPtwQZUMmATdTuA8/8l2jzBY1rVh/gwBDsw= -git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20231018083019-2b6d84de9a3d/go.mod h1:rQFJJdEOV7KbbMtQYR2lNfiZk+ONRDJSbMCTWxKt8Fw= +git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 h1:HeY8n27VyPRQe49l/fzyVMkWEB2fsLJYKp64pwA7tz4= +git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02/go.mod h1:rQFJJdEOV7KbbMtQYR2lNfiZk+ONRDJSbMCTWxKt8Fw= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= From 7ec9b34d33c08fdf340420c1096fc54c6e918292 Mon Sep 17 00:00:00 2001 From: Artem Tataurov Date: Fri, 16 Feb 2024 17:50:46 +0300 Subject: [PATCH 437/548] [#105] logger: Fix logging level changing for journald Signed-off-by: Artem Tataurov --- cmd/http-gw/settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 045a118..a38ebc5 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -423,7 +423,7 @@ func newJournaldLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields) - core := zapjournald.NewCore(zap.NewAtomicLevelAt(lvl), encoder, &journald.Journal{}, zapjournald.SyslogFields) + core := zapjournald.NewCore(c.Level, encoder, &journald.Journal{}, zapjournald.SyslogFields) coreWithContext := core.With([]zapcore.Field{ zapjournald.SyslogFacility(zapjournald.LogDaemon), zapjournald.SyslogIdentifier(), From 007d278caa2b80f3a7d780df7b95c81876916d5c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 29 Feb 2024 12:12:44 +0300 Subject: [PATCH 438/548] [#107] Close server listener on error Signed-off-by: Denis Kirillov --- cmd/http-gw/server.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/http-gw/server.go b/cmd/http-gw/server.go index c5852d8..f8a20d9 100644 --- a/cmd/http-gw/server.go +++ b/cmd/http-gw/server.go @@ -68,7 +68,8 @@ func newServer(ctx context.Context, serverInfo ServerInfo) (*server, error) { if serverInfo.TLS.Enabled { if err = tlsProvider.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { - return nil, fmt.Errorf("failed to update cert: %w", err) + lnErr := ln.Close() + return nil, fmt.Errorf("failed to update cert (listener close: %v): %w", lnErr, err) } ln = tls.NewListener(ln, &tls.Config{ From 88e32ddd7fe062822b04d1c07d9ed6cac6ebbe4b Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 29 Feb 2024 12:17:22 +0300 Subject: [PATCH 439/548] [#107] Add return on error in tokenizer middleware Signed-off-by: Denis Kirillov --- cmd/http-gw/app.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index aabb22b..4532520 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -549,8 +549,9 @@ func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { appCtx, err := tokens.StoreBearerTokenAppCtx(a.ctx, req) if err != nil { - a.log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err)) + a.log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Uint64("id", req.ID()), zap.Error(err)) response.Error(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) + return } utils.SetContextToRequest(appCtx, req) h(req) From 5ded105c09a7caff90d1e73f47d0ff314fd3f5d3 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 29 Feb 2024 12:50:56 +0300 Subject: [PATCH 440/548] [#107] Check query unescape errors Signed-off-by: Denis Kirillov --- internal/handler/download.go | 12 ++++++++++-- internal/handler/handler.go | 25 +++++++++++++++++++------ internal/logs/logs.go | 3 +-- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/internal/handler/download.go b/internal/handler/download.go index 06a247a..a7aee64 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -82,8 +82,16 @@ func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, // DownloadZipped handles zip by prefix requests. func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { scid, _ := c.UserValue("cid").(string) - prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string)) - log := h.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) + prefix, _ := c.UserValue("prefix").(string) + + prefix, err := url.QueryUnescape(prefix) + if err != nil { + h.log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Uint64("id", c.ID()), zap.Error(err)) + response.Error(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + log := h.log.With(zap.String("cid", scid), zap.String("prefix", prefix), zap.Uint64("id", c.ID())) ctx := utils.GetContextFromRequest(c) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 757b5be..f88dff1 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -131,12 +131,25 @@ func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, // byAttribute is a wrapper similar to byAddress. func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { - var ( - scid, _ = c.UserValue("cid").(string) - key, _ = url.QueryUnescape(c.UserValue("attr_key").(string)) - val, _ = url.QueryUnescape(c.UserValue("attr_val").(string)) - log = h.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) - ) + scid, _ := c.UserValue("cid").(string) + key, _ := c.UserValue("attr_key").(string) + val, _ := c.UserValue("attr_val").(string) + + key, err := url.QueryUnescape(key) + if err != nil { + h.log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_key", key), zap.Uint64("id", c.ID()), zap.Error(err)) + response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + val, err = url.QueryUnescape(val) + if err != nil { + h.log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_val", val), zap.Uint64("id", c.ID()), zap.Error(err)) + response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + log := h.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ctx := utils.GetContextFromRequest(c) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index c75e91f..84954c3 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -4,7 +4,6 @@ const ( CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/* CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go - WrongContainerID = "wrong container id" // Error in ../../downloader/download.go and uploader/upload.go WrongObjectID = "wrong object id" // Error in ../../downloader/download.go GetLatestObjectVersion = "get latest object version" // Error in ../../downloader/download.go ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go @@ -73,5 +72,5 @@ const ( InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go - + FailedToUnescapeQuery = "failed to unescape query" ) From c6383fc1355e8462d6dc1054f2a0cf7d5e29225c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 29 Feb 2024 12:52:52 +0300 Subject: [PATCH 441/548] [#107] Update CHANGELOG.md Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8022ef..f763e5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ See new `frostfs.tree_pool_max_attempts` config parameter. ### Fixed - Fix possibility of panic during SIGHUP (#99) +- Handle query unescape and invalid bearer token errors (#107) ### Added - Support client side object cut (#70) From 6695ebe5a0dd37900c786d9da28a13cf237a7cc2 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 27 Mar 2024 19:20:21 +0300 Subject: [PATCH 442/548] [#110] Test HTTP/2 requests Signed-off-by: Alex Vanin --- cmd/http-gw/server_test.go | 119 +++++++++++++++++++++++++++++++++++++ go.mod | 2 +- 2 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 cmd/http-gw/server_test.go diff --git a/cmd/http-gw/server_test.go b/cmd/http-gw/server_test.go new file mode 100644 index 0000000..a937366 --- /dev/null +++ b/cmd/http-gw/server_test.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "net/http" + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/net/http2" +) + +const ( + expHeaderKey = "Foo" + expHeaderValue = "Bar" +) + +func TestHTTP2TLS(t *testing.T) { + ctx := context.Background() + certPath, keyPath := prepareTestCerts(t) + + srv := &http.Server{ + Handler: http.HandlerFunc(testHandler), + } + + tlsListener, err := newServer(ctx, ServerInfo{ + Address: ":0", + TLS: ServerTLSInfo{ + Enabled: true, + CertFile: certPath, + KeyFile: keyPath, + }, + }) + require.NoError(t, err) + port := tlsListener.Listener().Addr().(*net.TCPAddr).Port + addr := fmt.Sprintf("https://localhost:%d", port) + + go func() { + _ = srv.Serve(tlsListener.Listener()) + }() + + // Server is running, now send HTTP/2 request + + tlsClientConfig := &tls.Config{ + InsecureSkipVerify: true, + } + + cliHTTP1 := http.Client{Transport: &http.Transport{TLSClientConfig: tlsClientConfig}} + cliHTTP2 := http.Client{Transport: &http2.Transport{TLSClientConfig: tlsClientConfig}} + + req, err := http.NewRequest("GET", addr, nil) + require.NoError(t, err) + req.Header[expHeaderKey] = []string{expHeaderValue} + + resp, err := cliHTTP1.Do(req) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + + resp, err = cliHTTP2.Do(req) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) +} + +func testHandler(resp http.ResponseWriter, req *http.Request) { + hdr, ok := req.Header[expHeaderKey] + if !ok || len(hdr) != 1 || hdr[0] != expHeaderValue { + resp.WriteHeader(http.StatusBadRequest) + } else { + resp.WriteHeader(http.StatusOK) + } +} + +func prepareTestCerts(t *testing.T) (certPath, keyPath string) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour * 24 * 365), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + require.NoError(t, err) + + dir := t.TempDir() + certPath = path.Join(dir, "cert.pem") + keyPath = path.Join(dir, "key.pem") + + certFile, err := os.Create(certPath) + require.NoError(t, err) + defer certFile.Close() + + keyFile, err := os.Create(keyPath) + require.NoError(t, err) + defer keyFile.Close() + + err = pem.Encode(certFile, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + require.NoError(t, err) + + err = pem.Encode(keyFile, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}) + require.NoError(t, err) + + return certPath, keyPath +} diff --git a/go.mod b/go.mod index 9388eae..dd2896c 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc + golang.org/x/net v0.10.0 google.golang.org/grpc v1.55.0 ) @@ -103,7 +104,6 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect From f39b3aa93a3e590f8e4fba415180e3e99f02843e Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 27 Mar 2024 19:20:49 +0300 Subject: [PATCH 443/548] [#110] Add "h2" as next proto to allow HTTP/2 requests in http.Serve Signed-off-by: Alex Vanin --- cmd/http-gw/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/http-gw/server.go b/cmd/http-gw/server.go index f8a20d9..694e9ee 100644 --- a/cmd/http-gw/server.go +++ b/cmd/http-gw/server.go @@ -74,6 +74,7 @@ func newServer(ctx context.Context, serverInfo ServerInfo) (*server, error) { ln = tls.NewListener(ln, &tls.Config{ GetCertificate: tlsProvider.GetCertificate, + NextProtos: []string{"h2"}, // required to enable HTTP/2 requests in `http.Serve` }) } From a95dc6c8c7d690259ea47a723e8a93db0eb193b6 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 27 Mar 2024 19:26:37 +0300 Subject: [PATCH 444/548] [#110] Update CHANGELOG Signed-off-by: Alex Vanin --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f763e5a..6da67f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ See new `frostfs.tree_pool_max_attempts` config parameter. ### Fixed - Fix possibility of panic during SIGHUP (#99) - Handle query unescape and invalid bearer token errors (#107) +- Fix HTTP/2 requests (#110) ### Added - Support client side object cut (#70) From 11965deb4188e78bdbe92b5d4b17e41620f2a86d Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Tue, 26 Mar 2024 14:34:20 +0300 Subject: [PATCH 445/548] [#100] server auto re-binding Signed-off-by: Pavel Pogodaev --- CHANGELOG.md | 1 + cmd/http-gw/app.go | 132 +++++++++++++++++++++++++++++++------ cmd/http-gw/settings.go | 21 +++++- config/config.env | 3 + config/config.yaml | 1 + docs/gate-configuration.md | 2 + internal/logs/logs.go | 4 ++ 7 files changed, 142 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6da67f5..105ac41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ This document outlines major changes between releases. ### Added - Tree pool traversal limit (#92) +- Add new `reconnect_interval` config param (#100) ### Update from 0.28.0 See new `frostfs.tree_pool_max_attempts` config parameter. diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 4532520..2a20d86 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "net/http" "os" @@ -57,7 +58,10 @@ type ( metrics *gateMetrics services []*metrics.Service settings *appSettings - servers []Server + + servers []Server + unbindServers []ServerInfo + mu sync.RWMutex } // App is an interface for the main gateway function. @@ -78,6 +82,8 @@ type ( // appSettings stores reloading parameters, so it has to provide getters and setters which use RWMutex. appSettings struct { + reconnectInterval time.Duration + mu sync.RWMutex defaultTimestamp bool zipCompression bool @@ -199,8 +205,9 @@ func (s *appSettings) setBufferMaxSizeForPut(val uint64) { } func (a *app) initAppSettings() { - a.settings = &appSettings{} - + a.settings = &appSettings{ + reconnectInterval: fetchReconnectInterval(a.cfg), + } a.updateSettings() } @@ -399,16 +406,22 @@ func (a *app) Serve() { a.startServices() a.initServers(a.ctx) - for i := range a.servers { + servs := a.getServers() + + for i := range servs { go func(i int) { - a.log.Info(logs.StartingServer, zap.String("address", a.servers[i].Address())) - if err := a.webServer.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed { - a.metrics.MarkUnhealthy(a.servers[i].Address()) + a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address())) + if err := a.webServer.Serve(servs[i].Listener()); err != nil && err != http.ErrServerClosed { + a.metrics.MarkUnhealthy(servs[i].Address()) a.log.Fatal(logs.ListenAndServe, zap.Error(err)) } }(i) } + if len(a.unbindServers) != 0 { + a.scheduleReconnect(a.ctx, a.webServer) + } + sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGHUP) @@ -598,7 +611,7 @@ func (a *app) AppParams() *utils.AppParams { } func (a *app) initServers(ctx context.Context) { - serversInfo := fetchServers(a.cfg) + serversInfo := fetchServers(a.cfg, a.log) a.servers = make([]Server, 0, len(serversInfo)) for _, serverInfo := range serversInfo { @@ -608,6 +621,7 @@ func (a *app) initServers(ctx context.Context) { } srv, err := newServer(ctx, serverInfo) if err != nil { + a.unbindServers = append(a.unbindServers, serverInfo) a.metrics.MarkUnhealthy(serverInfo.Address) a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...) continue @@ -624,21 +638,24 @@ func (a *app) initServers(ctx context.Context) { } func (a *app) updateServers() error { - serversInfo := fetchServers(a.cfg) + serversInfo := fetchServers(a.cfg, a.log) + + a.mu.Lock() + defer a.mu.Unlock() var found bool for _, serverInfo := range serversInfo { - index := a.serverIndex(serverInfo.Address) - if index == -1 { - continue - } - - if serverInfo.TLS.Enabled { - if err := a.servers[index].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { - return fmt.Errorf("failed to update tls certs: %w", err) + ser := a.getServer(serverInfo.Address) + if ser != nil { + if serverInfo.TLS.Enabled { + if err := ser.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { + return fmt.Errorf("failed to update tls certs: %w", err) + } + found = true } + } else if unbind := a.updateUnbindServerInfo(serverInfo); unbind { + found = true } - found = true } if !found { @@ -648,13 +665,29 @@ func (a *app) updateServers() error { return nil } -func (a *app) serverIndex(address string) int { +func (a *app) getServers() []Server { + a.mu.RLock() + defer a.mu.RUnlock() + return a.servers +} + +func (a *app) getServer(address string) Server { for i := range a.servers { if a.servers[i].Address() == address { - return i + return a.servers[i] } } - return -1 + return nil +} + +func (a *app) updateUnbindServerInfo(info ServerInfo) bool { + for i := range a.unbindServers { + if a.unbindServers[i].Address == info.Address { + a.unbindServers[i] = info + return true + } + } + return false } func (a *app) initTracing(ctx context.Context) { @@ -727,3 +760,60 @@ func (s *appSettings) setDefaultNamespaces(namespaces []string) { s.defaultNamespaces = namespaces s.mu.Unlock() } + +func (a *app) scheduleReconnect(ctx context.Context, srv *fasthttp.Server) { + go func() { + t := time.NewTicker(a.settings.reconnectInterval) + defer t.Stop() + for { + select { + case <-t.C: + if a.tryReconnect(ctx, srv) { + return + } + t.Reset(a.settings.reconnectInterval) + case <-ctx.Done(): + return + } + } + }() +} + +func (a *app) tryReconnect(ctx context.Context, sr *fasthttp.Server) bool { + a.mu.Lock() + defer a.mu.Unlock() + + a.log.Info(logs.ServerReconnecting) + var failedServers []ServerInfo + + for _, serverInfo := range a.unbindServers { + fields := []zap.Field{ + zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled), + zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile), + } + + srv, err := newServer(ctx, serverInfo) + if err != nil { + a.log.Warn(logs.ServerReconnectFailed, zap.Error(err)) + failedServers = append(failedServers, serverInfo) + a.metrics.MarkUnhealthy(serverInfo.Address) + continue + } + + go func() { + a.log.Info(logs.StartingServer, zap.String("address", srv.Address())) + a.metrics.MarkHealthy(serverInfo.Address) + if err = sr.Serve(srv.Listener()); err != nil && !errors.Is(err, http.ErrServerClosed) { + a.log.Warn(logs.ListenAndServe, zap.Error(err)) + a.metrics.MarkUnhealthy(serverInfo.Address) + } + }() + + a.servers = append(a.servers, srv) + a.log.Info(logs.ServerReconnectedSuccessfully, fields...) + } + + a.unbindServers = failedServers + + return len(a.unbindServers) == 0 +} diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index a38ebc5..0d97dcb 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -51,11 +51,15 @@ const ( defaultNamespaceHeader = "X-Frostfs-Namespace" + defaultReconnectInterval = time.Minute + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" cfgTLSKeyFile = "tls.key_file" + cfgReconnectInterval = "reconnect_interval" + // Web. cfgWebReadBufferSize = "web.read_buffer_size" cfgWebWriteBufferSize = "web.write_buffer_size" @@ -454,8 +458,18 @@ func getLogLevel(v *viper.Viper) (zapcore.Level, error) { return lvl, nil } -func fetchServers(v *viper.Viper) []ServerInfo { +func fetchReconnectInterval(cfg *viper.Viper) time.Duration { + reconnect := cfg.GetDuration(cfgReconnectInterval) + if reconnect <= 0 { + reconnect = defaultReconnectInterval + } + + return reconnect +} + +func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { var servers []ServerInfo + seen := make(map[string]struct{}) for i := 0; ; i++ { key := cfgServer + "." + strconv.Itoa(i) + "." @@ -470,6 +484,11 @@ func fetchServers(v *viper.Viper) []ServerInfo { break } + if _, ok := seen[serverInfo.Address]; ok { + log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address)) + continue + } + seen[serverInfo.Address] = struct{}{} servers = append(servers, serverInfo) } diff --git a/config/config.env b/config/config.env index 12f1ba4..05b83b3 100644 --- a/config/config.env +++ b/config/config.env @@ -26,6 +26,9 @@ HTTP_GW_SERVER_1_TLS_ENABLED=true HTTP_GW_SERVER_1_TLS_CERT_FILE=/path/to/tls/cert HTTP_GW_SERVER_1_TLS_KEY_FILE=/path/to/tls/key +# How often to reconnect to the servers +HTTP_GW_RECONNECT_INTERVAL: 1m + # Nodes configuration. # This configuration make the gateway use the first node (grpc://s01.frostfs.devenv:8080) # while it's healthy. Otherwise, the gateway use the second node (grpc://s01.frostfs.devenv:8080) diff --git a/config/config.yaml b/config/config.yaml index 7ea2748..7f8077b 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -55,6 +55,7 @@ peers: priority: 2 weight: 9 +reconnect_interval: 1m web: # Per-connection buffer size for requests' reading. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index bf792b7..8e3daad 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -72,6 +72,7 @@ stream_timeout: 10s request_timeout: 5s rebalance_timer: 30s pool_error_threshold: 100 +reconnect_interval: 1m ``` | Parameter | Type | SIGHUP reload | Default value | Description | @@ -83,6 +84,7 @@ pool_error_threshold: 100 | `request_timeout` | `duration` | | `15s` | Timeout to check node health during rebalance. | | `rebalance_timer` | `duration` | | `60s` | Interval to check node health. | | `pool_error_threshold` | `uint32` | | `100` | The number of errors on connection after which node is considered as unhealthy. | +| `reconnect_interval` | `duration` | no | `1m` | Listeners reconnection interval. | # `wallet` section diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 84954c3..0ab5dbf 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -73,4 +73,8 @@ const ( InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go FailedToUnescapeQuery = "failed to unescape query" + ServerReconnecting = "reconnecting server..." + ServerReconnectedSuccessfully = "server reconnected successfully" + ServerReconnectFailed = "failed to reconnect server" + WarnDuplicateAddress = "duplicate address" ) From 16d6e6c34e3676330118d89e3e108348aea76d64 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Fri, 3 May 2024 15:29:51 +0300 Subject: [PATCH 446/548] [#112] tokens: Extend test coverage Signed-off-by: Roman Loginov --- tokens/bearer-token_test.go | 193 +++++++++++++++++++++++++++++++----- 1 file changed, 168 insertions(+), 25 deletions(-) diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index cc54e74..6fb3bf4 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -23,19 +23,29 @@ func makeTestCookie(value []byte) *fasthttp.RequestHeader { func makeTestHeader(value []byte) *fasthttp.RequestHeader { header := new(fasthttp.RequestHeader) if value != nil { - header.Set(fasthttp.HeaderAuthorization, bearerTokenHdr+" "+string(value)) + header.Set(fasthttp.HeaderAuthorization, string(value)) } return header } -func Test_fromCookie(t *testing.T) { +func makeBearer(value string) string { + return bearerTokenHdr + " " + value +} + +func TestBearerTokenFromCookie(t *testing.T) { cases := []struct { name string actual []byte expect []byte }{ - {name: "empty"}, - {name: "normal", actual: []byte("TOKEN"), expect: []byte("TOKEN")}, + { + name: "empty", + }, + { + name: "normal", + actual: []byte("TOKEN"), + expect: []byte("TOKEN"), + }, } for _, tt := range cases { @@ -45,14 +55,31 @@ func Test_fromCookie(t *testing.T) { } } -func Test_fromHeader(t *testing.T) { +func TestBearerTokenFromHeader(t *testing.T) { + validToken := "token" + tokenWithoutPrefix := "invalid-token" + cases := []struct { name string actual []byte expect []byte }{ - {name: "empty"}, - {name: "normal", actual: []byte("TOKEN"), expect: []byte("TOKEN")}, + { + name: "empty", + }, + { + name: "token without the bearer prefix", + actual: []byte(tokenWithoutPrefix), + }, + { + name: "token without payload", + actual: []byte(makeBearer("")), + }, + { + name: "normal", + actual: []byte(makeBearer(validToken)), + expect: []byte(validToken), + }, } for _, tt := range cases { @@ -62,7 +89,7 @@ func Test_fromHeader(t *testing.T) { } } -func Test_fetchBearerToken(t *testing.T) { +func TestFetchBearerToken(t *testing.T) { key, err := keys.NewPrivateKey() require.NoError(t, err) var uid user.ID @@ -75,43 +102,77 @@ func Test_fetchBearerToken(t *testing.T) { require.NotEmpty(t, t64) cases := []struct { - name string - + name string cookie string header string - error string + nilCtx bool expect *bearer.Token }{ - {name: "empty"}, - - {name: "bad base64 header", header: "WRONG BASE64", error: "can't base64-decode bearer token"}, - {name: "bad base64 cookie", cookie: "WRONG BASE64", error: "can't base64-decode bearer token"}, - - {name: "header token unmarshal error", header: "dGVzdAo=", error: "can't unmarshal bearer token"}, - {name: "cookie token unmarshal error", cookie: "dGVzdAo=", error: "can't unmarshal bearer token"}, - + { + name: "empty", + }, + { + name: "nil context", + nilCtx: true, + }, + { + name: "bad base64 header", + header: "WRONG BASE64", + error: "can't base64-decode bearer token", + }, + { + name: "bad base64 cookie", + cookie: "WRONG BASE64", + error: "can't base64-decode bearer token", + }, + { + name: "header token unmarshal error", + header: "dGVzdAo=", + error: "can't unmarshal bearer token", + }, + { + name: "cookie token unmarshal error", + cookie: "dGVzdAo=", + error: "can't unmarshal bearer token", + }, { name: "bad header and cookie", header: "WRONG BASE64", cookie: "dGVzdAo=", error: "can't unmarshal bearer token", }, - { name: "bad header, but good cookie", header: "dGVzdAo=", cookie: t64, expect: tkn, }, - - {name: "ok for header", header: t64, expect: tkn}, - {name: "ok for cookie", cookie: t64, expect: tkn}, + { + name: "bad cookie, but good header", + header: t64, + cookie: "dGVzdAo=", + expect: tkn, + }, + { + name: "ok for header", + header: t64, + expect: tkn, + }, + { + name: "ok for cookie", + cookie: t64, + expect: tkn, + }, } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - ctx := makeTestRequest(tt.cookie, tt.header) + var ctx *fasthttp.RequestCtx + if !tt.nilCtx { + ctx = makeTestRequest(tt.cookie, tt.header) + } + actual, err := fetchBearerToken(ctx) if tt.error == "" { @@ -139,7 +200,7 @@ func makeTestRequest(cookie, header string) *fasthttp.RequestCtx { return ctx } -func Test_checkAndPropagateBearerToken(t *testing.T) { +func TestCheckAndPropagateBearerToken(t *testing.T) { key, err := keys.NewPrivateKey() require.NoError(t, err) var uid user.ID @@ -162,3 +223,85 @@ func Test_checkAndPropagateBearerToken(t *testing.T) { require.NoError(t, err) require.Equal(t, tkn, actual) } + +func TestLoadBearerToken(t *testing.T) { + ctx := context.Background() + token := new(bearer.Token) + + cases := []struct { + name string + appCtx context.Context + error string + }{ + { + name: "token is missing in the context", + appCtx: ctx, + error: "found empty bearer token", + }, + { + name: "normal", + appCtx: context.WithValue(ctx, bearerTokenKey, token), + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + tkn, err := LoadBearerToken(tt.appCtx) + + if tt.error == "" { + require.NoError(t, err) + require.Equal(t, token, tkn) + + return + } + + require.Contains(t, err.Error(), tt.error) + }) + } +} + +func TestStoreBearerTokenAppCtx(t *testing.T) { + key, err := keys.NewPrivateKey() + require.NoError(t, err) + var uid user.ID + user.IDFromKey(&uid, key.PrivateKey.PublicKey) + + tkn := new(bearer.Token) + tkn.ForUser(uid) + + t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) + require.NotEmpty(t, t64) + + cases := []struct { + name string + req *fasthttp.RequestCtx + error string + }{ + { + name: "invalid token", + req: makeTestRequest("dGVzdAo=", ""), + error: "can't unmarshal bearer token", + }, + { + name: "normal", + req: makeTestRequest(t64, ""), + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + ctx, err := StoreBearerTokenAppCtx(context.Background(), tt.req) + + if tt.error == "" { + require.NoError(t, err) + actualToken, ok := ctx.Value(bearerTokenKey).(*bearer.Token) + require.True(t, ok) + require.Equal(t, tkn, actualToken) + + return + } + + require.Contains(t, err.Error(), tt.error) + }) + } +} From c851c0529c134021189ba7437dccd7a7fea22e65 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Mon, 6 May 2024 20:34:49 +0300 Subject: [PATCH 447/548] [#112] Add integration test with bearer token Signed-off-by: Roman Loginov --- cmd/http-gw/integration_test.go | 94 +++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index f76c3ce..5d21094 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -6,25 +6,30 @@ import ( "archive/zip" "bytes" "context" + "encoding/base64" "encoding/json" "fmt" "io" "mime/multipart" "net/http" + "os" "sort" "testing" "time" containerv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/viper" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" @@ -47,11 +52,17 @@ func TestIntegration(t *testing.T) { rootCtx := context.Background() aioImage := "truecloudlab/frostfs-aio:" versions := []string{ - "1.2.7", // frostfs-storage v0.36.0 RC + "1.2.7", + "1.3.0", } key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) + file, err := os.CreateTemp("", "wallet") + require.NoError(t, err) + defer os.Remove(file.Name()) + makeTempWallet(t, key, file.Name()) + var ownerID user.ID user.IDFromKey(&ownerID, key.PrivateKey.PublicKey) @@ -59,12 +70,16 @@ func TestIntegration(t *testing.T) { ctx, cancel2 := context.WithCancel(rootCtx) aioContainer := createDockerContainer(ctx, t, aioImage+version) - server, cancel := runServer() + server, cancel := runServer(file.Name()) clientPool := getPool(ctx, t, key) CID, err := createContainer(ctx, t, clientPool, ownerID, version) require.NoError(t, err, version) + token := makeBearerToken(t, key, ownerID, version) + t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID, version) }) + t.Run("put with bearer token in header"+version, func(t *testing.T) { putWithBearerTokenInHeader(ctx, t, clientPool, CID, token) }) + t.Run("put with bearer token in cookie"+version, func(t *testing.T) { putWithBearerTokenInCookie(ctx, t, clientPool, CID, token) }) t.Run("put with duplicate keys "+version, func(t *testing.T) { putWithDuplicateKeys(t, CID) }) t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, ownerID, CID, version) }) t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID, version) }) @@ -79,10 +94,13 @@ func TestIntegration(t *testing.T) { } } -func runServer() (App, context.CancelFunc) { +func runServer(pathToWallet string) (App, context.CancelFunc) { cancelCtx, cancel := context.WithCancel(context.Background()) v := getDefaultConfig() + v.Set(cfgWalletPath, pathToWallet) + v.Set(cfgWalletPassphrase, "") + l, lvl := newStdoutLogger(zapcore.DebugLevel) application := newApp(cancelCtx, WithConfig(v), WithLogger(l, lvl)) go application.Serve() @@ -98,7 +116,38 @@ func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID, vers makePutRequestAndCheck(ctx, t, p, CID, url) } +func putWithBearerTokenInHeader(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID, token string) { + url := testHost + "/upload/" + CID.String() + + request, content, attributes := makePutRequest(t, url) + request.Header.Set("Authorization", "Bearer "+token) + resp, err := http.DefaultClient.Do(request) + require.NoError(t, err) + + checkPutResponse(ctx, t, p, CID, resp, content, attributes) +} + +func putWithBearerTokenInCookie(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID, token string) { + url := testHost + "/upload/" + CID.String() + + request, content, attributes := makePutRequest(t, url) + request.AddCookie(&http.Cookie{Name: "Bearer", Value: token}) + resp, err := http.DefaultClient.Do(request) + require.NoError(t, err) + + checkPutResponse(ctx, t, p, CID, resp, content, attributes) +} + func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID cid.ID, url string) { + request, content, attributes := makePutRequest(t, url) + + resp, err := http.DefaultClient.Do(request) + require.NoError(t, err) + + checkPutResponse(ctx, t, p, cnrID, resp, content, attributes) +} + +func makePutRequest(t *testing.T, url string) (*http.Request, string, map[string]string) { content := "content of file" keyAttr, valAttr := "User-Attribute", "user value" attributes := map[string]string{ @@ -120,9 +169,10 @@ func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnr request.Header.Set("Content-Type", w.FormDataContentType()) request.Header.Set("X-Attribute-"+keyAttr, valAttr) - resp, err := http.DefaultClient.Do(request) - require.NoError(t, err) + return request, content, attributes +} +func checkPutResponse(ctx context.Context, t *testing.T, p *pool.Pool, cnrID cid.ID, resp *http.Response, content string, attributes map[string]string) { defer func() { err := resp.Body.Close() require.NoError(t, err) @@ -476,3 +526,37 @@ func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID return id } + +func makeBearerToken(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string) string { + tkn := new(bearer.Token) + tkn.ForUser(ownerID) + tkn.SetExp(10000) + + if version == "1.2.7" { + tkn.SetEACLTable(*eacl.NewTable()) + } else { + tkn.SetImpersonate(true) + } + + err := tkn.Sign(key.PrivateKey) + require.NoError(t, err) + + t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) + require.NotEmpty(t, t64) + + return t64 +} + +func makeTempWallet(t *testing.T, key *keys.PrivateKey, path string) { + w, err := wallet.NewWallet(path) + require.NoError(t, err) + + acc := wallet.NewAccountFromPrivateKey(key) + err = acc.Encrypt("", w.Scrypt) + require.NoError(t, err) + + w.AddAccount(acc) + + err = w.Save() + require.NoError(t, err) +} From 5b7b872dcd507f78e2b4ac317973d2ddf9e031f2 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Tue, 7 May 2024 17:58:41 +0300 Subject: [PATCH 448/548] [#112] Update net to v0.23.0 Signed-off-by: Roman Loginov --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index dd2896c..d862839 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc - golang.org/x/net v0.10.0 + golang.org/x/net v0.23.0 google.golang.org/grpc v1.55.0 ) @@ -103,11 +103,11 @@ require ( go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.9.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/sync v0.2.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/term v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.30.0 // indirect diff --git a/go.sum b/go.sum index 1bd67bc..be1f16d 100644 --- a/go.sum +++ b/go.sum @@ -1024,8 +1024,8 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1118,8 +1118,8 @@ golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1245,13 +1245,13 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1261,8 +1261,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From b73a4a25b37a291679de7cddcfbc21832a71665d Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 5 Jun 2024 12:29:55 +0300 Subject: [PATCH 449/548] [#115] go.mod: Update vulnerable dependencies Signed-off-by: Denis Kirillov --- go.mod | 14 ++++++++------ go.sum | 35 ++++++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index d862839..f30c263 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw -go 1.20 +go 1.21 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4 @@ -23,7 +23,7 @@ require ( go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc golang.org/x/net v0.23.0 - google.golang.org/grpc v1.55.0 + google.golang.org/grpc v1.61.1 ) require ( @@ -55,7 +55,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect @@ -104,13 +104,15 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sync v0.2.0 // indirect + golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index be1f16d..b372c25 100644 --- a/go.sum +++ b/go.sum @@ -241,6 +241,7 @@ github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/continuity v0.2.2 h1:QSqfxcn8c+12slxwu00AtzXrsami0MJb/MQs9lOLHLA= +github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= @@ -374,6 +375,7 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -436,7 +438,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -492,7 +495,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -515,8 +519,9 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= @@ -607,6 +612,7 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -828,6 +834,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -995,6 +1002,7 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -1144,8 +1152,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1410,8 +1418,12 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1437,8 +1449,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY= +google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1453,8 +1465,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1464,6 +1476,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= From 5a87ee7625b8bc12eaab88ce0bbaf5582e04b1fd Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 5 Jun 2024 14:00:01 +0300 Subject: [PATCH 450/548] [#115] Fix ci build go version Signed-off-by: Denis Kirillov --- .docker/Dockerfile | 2 +- .forgejo/workflows/builds.yml | 2 +- .forgejo/workflows/tests.yml | 4 ++-- .forgejo/workflows/vulncheck.yml | 2 +- Makefile | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 8b450a4..d39fba1 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21-alpine as basebuilder +FROM golang:1.22-alpine as basebuilder RUN apk add --update make bash ca-certificates FROM basebuilder as builder diff --git a/.forgejo/workflows/builds.yml b/.forgejo/workflows/builds.yml index 97ac86b..17f1f2e 100644 --- a/.forgejo/workflows/builds.yml +++ b/.forgejo/workflows/builds.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.20', '1.21' ] + go_versions: [ '1.21', '1.22' ] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 14b9edf..74e0b2c 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' cache: true - name: Install linters @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.20', '1.21' ] + go_versions: [ '1.21', '1.22' ] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 0139e89..7a82bc3 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/Makefile b/Makefile index d02d41b..04cfea4 100755 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") -GO_VERSION ?= 1.20 +GO_VERSION ?= 1.22 LINT_VERSION ?= 1.54.0 TRUECLOUDLAB_LINT_VERSION ?= 0.0.2 BUILD ?= $(shell date -u --iso=seconds) From 23ed3ab86e68c275f17fb7956459a818b72bfd27 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Mon, 3 Jun 2024 01:42:01 +0300 Subject: [PATCH 451/548] [#114] Update frostfs-sdk-go version with support EC Signed-off-by: Roman Loginov --- go.mod | 4 ++-- go.sum | 8 ++++---- internal/handler/upload.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index f30c263..a34a092 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.21 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 github.com/fasthttp/router v1.4.1 diff --git a/go.sum b/go.sum index b372c25..0cde076 100644 --- a/go.sum +++ b/go.sum @@ -37,16 +37,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4 h1:wjLfZ3WCt7qNGsQv+Jl0TXnmtg0uVk/jToKPFTBc/jo= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 h1:H5GvrVlowIMWfzqQkhY0p0myooJxQ1sMRVSFfXawwWg= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939 h1:jZEepi9yWmqrWgLRQcHQu4YPJaudmd7d2AEhpmM3m4U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231107114540-ab75edd70939/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f h1:vBLC1OSGMSn7lRJv/p1of0veifuBdZdztVrF9Vn+UFk= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 935b51b..1b18755 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -129,7 +129,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { obj := object.New() obj.SetContainerID(bktInfo.CID) - obj.SetOwnerID(h.ownerID) + obj.SetOwnerID(*h.ownerID) obj.SetAttributes(attributes...) var prm pool.PrmObjectPut From 826dd0cdbe2511f505e06e44dcf7696119a8d6fe Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 14 Jun 2024 16:43:58 +0300 Subject: [PATCH 452/548] [#117] Fix integration test after updating dependencies Signed-off-by: Denis Kirillov --- cmd/http-gw/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 5d21094..cae40a5 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -506,7 +506,7 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, content string, attributes map[string]string) oid.ID { obj := object.New() obj.SetContainerID(CID) - obj.SetOwnerID(&ownerID) + obj.SetOwnerID(ownerID) var attrs []object.Attribute for key, val := range attributes { From 3741e3b003e40715c258f17638da88c43c1d92df Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 17 Jun 2024 17:56:43 +0300 Subject: [PATCH 453/548] [#117] Add mocked handler for tests Signed-off-by: Denis Kirillov --- cmd/http-gw/app.go | 7 +- internal/frostfs/frostfs.go | 260 +++++++++++++++++++++++++++ internal/handler/download.go | 31 ++-- internal/handler/frostfs_mock.go | 260 +++++++++++++++++++++++++++ internal/handler/handler.go | 124 +++++++++++-- internal/handler/handler_test.go | 296 +++++++++++++++++++++++++++++++ internal/handler/head.go | 37 ++-- internal/handler/reader.go | 20 ++- internal/handler/upload.go | 24 ++- utils/attributes.go | 16 +- utils/params.go | 17 -- utils/util.go | 26 --- 12 files changed, 1005 insertions(+), 113 deletions(-) create mode 100644 internal/frostfs/frostfs.go create mode 100644 internal/handler/frostfs_mock.go create mode 100644 internal/handler/handler_test.go delete mode 100644 utils/params.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 2a20d86..1d12122 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -16,6 +16,7 @@ import ( v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" @@ -600,10 +601,10 @@ func (a *app) reqNamespace(h fasthttp.RequestHandler) fasthttp.RequestHandler { } } -func (a *app) AppParams() *utils.AppParams { - return &utils.AppParams{ +func (a *app) AppParams() *handler.AppParams { + return &handler.AppParams{ Logger: a.log, - Pool: a.pool, + FrostFS: frostfs.NewFrostFS(a.pool), Owner: a.owner, Resolver: a.resolver, Cache: cache.NewBucketCache(getCacheOptions(a.cfg, a.log)), diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go new file mode 100644 index 0000000..dde560b --- /dev/null +++ b/internal/frostfs/frostfs.go @@ -0,0 +1,260 @@ +package frostfs + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// FrostFS represents virtual connection to the FrostFS network. +// It is used to provide an interface to dependent packages +// which work with FrostFS. +type FrostFS struct { + pool *pool.Pool +} + +// NewFrostFS creates new FrostFS using provided pool.Pool. +func NewFrostFS(p *pool.Pool) *FrostFS { + return &FrostFS{ + pool: p, + } +} + +// Container implements frostfs.FrostFS interface method. +func (x *FrostFS) Container(ctx context.Context, layerPrm handler.PrmContainer) (*container.Container, error) { + prm := pool.PrmContainerGet{ + ContainerID: layerPrm.ContainerID, + } + + res, err := x.pool.GetContainer(ctx, prm) + if err != nil { + return nil, handleObjectError("read container via connection pool", err) + } + + return &res, nil +} + +// CreateObject implements frostfs.FrostFS interface method. +func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) (oid.ID, error) { + var prmPut pool.PrmObjectPut + prmPut.SetHeader(*prm.Object) + prmPut.SetPayload(prm.Payload) + prmPut.SetClientCut(prm.ClientCut) + prmPut.WithoutHomomorphicHash(prm.WithoutHomomorphicHash) + prmPut.SetBufferMaxSize(prm.BufferMaxSize) + + if prm.BearerToken != nil { + prmPut.UseBearer(*prm.BearerToken) + } + + idObj, err := x.pool.PutObject(ctx, prmPut) + return idObj, handleObjectError("save object via connection pool", err) +} + +// wraps io.ReadCloser and transforms Read errors related to access violation +// to frostfs.ErrAccessDenied. +type payloadReader struct { + io.ReadCloser +} + +func (x payloadReader) Read(p []byte) (int, error) { + n, err := x.ReadCloser.Read(p) + if err != nil && errors.Is(err, io.EOF) { + return n, err + } + return n, handleObjectError("read payload", err) +} + +// ReadObject implements frostfs.FrostFS interface method. +func (x *FrostFS) ReadObject(ctx context.Context, prm handler.PrmObjectRead) (*handler.ObjectPart, error) { + var prmGet pool.PrmObjectGet + prmGet.SetAddress(prm.Address) + + if prm.BearerToken != nil { + prmGet.UseBearer(*prm.BearerToken) + } + + if prm.WithHeader { + if prm.WithPayload { + res, err := x.pool.GetObject(ctx, prmGet) + if err != nil { + return nil, handleObjectError("init full object reading via connection pool", err) + } + + defer res.Payload.Close() + + payload, err := io.ReadAll(res.Payload) + if err != nil { + return nil, handleObjectError("read full object payload", err) + } + + res.Header.SetPayload(payload) + + return &handler.ObjectPart{ + Head: &res.Header, + }, nil + } + + var prmHead pool.PrmObjectHead + prmHead.SetAddress(prm.Address) + + if prm.BearerToken != nil { + prmHead.UseBearer(*prm.BearerToken) + } + + hdr, err := x.pool.HeadObject(ctx, prmHead) + if err != nil { + return nil, handleObjectError("read object header via connection pool", err) + } + + return &handler.ObjectPart{ + Head: &hdr, + }, nil + } else if prm.PayloadRange[0]+prm.PayloadRange[1] == 0 { + res, err := x.pool.GetObject(ctx, prmGet) + if err != nil { + return nil, handleObjectError("init full payload range reading via connection pool", err) + } + + return &handler.ObjectPart{ + Payload: res.Payload, + }, nil + } + + var prmRange pool.PrmObjectRange + prmRange.SetAddress(prm.Address) + prmRange.SetOffset(prm.PayloadRange[0]) + prmRange.SetLength(prm.PayloadRange[1]) + + if prm.BearerToken != nil { + prmRange.UseBearer(*prm.BearerToken) + } + + res, err := x.pool.ObjectRange(ctx, prmRange) + if err != nil { + return nil, handleObjectError("init payload range reading via connection pool", err) + } + + return &handler.ObjectPart{ + Payload: payloadReader{&res}, + }, nil +} + +// SearchObjects implements frostfs.FrostFS interface method. +func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch) (handler.ResObjectSearch, error) { + var prmSearch pool.PrmObjectSearch + prmSearch.SetContainerID(prm.Container) + prmSearch.SetFilters(prm.Filters) + + if prm.BearerToken != nil { + prmSearch.UseBearer(*prm.BearerToken) + } + + res, err := x.pool.SearchObjects(ctx, prmSearch) + if err != nil { + return nil, handleObjectError("init object search via connection pool", err) + } + + return &res, nil +} + +// GetEpochDurations implements frostfs.FrostFS interface method. +func (x *FrostFS) GetEpochDurations(ctx context.Context) (*utils.EpochDurations, error) { + networkInfo, err := x.pool.NetworkInfo(ctx) + if err != nil { + return nil, err + } + + res := &utils.EpochDurations{ + CurrentEpoch: networkInfo.CurrentEpoch(), + MsPerBlock: networkInfo.MsPerBlock(), + BlockPerEpoch: networkInfo.EpochDuration(), + } + + if res.BlockPerEpoch == 0 { + return nil, fmt.Errorf("EpochDuration is empty") + } + return res, nil +} + +// ResolverFrostFS represents virtual connection to the FrostFS network. +// It implements resolver.FrostFS. +type ResolverFrostFS struct { + pool *pool.Pool +} + +// NewResolverFrostFS creates new ResolverFrostFS using provided pool.Pool. +func NewResolverFrostFS(p *pool.Pool) *ResolverFrostFS { + return &ResolverFrostFS{pool: p} +} + +// SystemDNS implements resolver.FrostFS interface method. +func (x *ResolverFrostFS) SystemDNS(ctx context.Context) (string, error) { + networkInfo, err := x.pool.NetworkInfo(ctx) + if err != nil { + return "", handleObjectError("read network info via client", err) + } + + domain := networkInfo.RawNetworkParameter("SystemDNS") + if domain == nil { + return "", errors.New("system DNS parameter not found or empty") + } + + return string(domain), nil +} + +func handleObjectError(msg string, err error) error { + if err == nil { + return nil + } + + if reason, ok := IsErrObjectAccessDenied(err); ok { + return fmt.Errorf("%s: %w: %s", msg, handler.ErrAccessDenied, reason) + } + + if IsTimeoutError(err) { + return fmt.Errorf("%s: %w: %s", msg, handler.ErrGatewayTimeout, err.Error()) + } + + return fmt.Errorf("%s: %w", msg, err) +} + +func UnwrapErr(err error) error { + unwrappedErr := errors.Unwrap(err) + for unwrappedErr != nil { + err = unwrappedErr + unwrappedErr = errors.Unwrap(err) + } + + return err +} + +func IsErrObjectAccessDenied(err error) (string, bool) { + err = UnwrapErr(err) + switch err := err.(type) { + default: + return "", false + case *apistatus.ObjectAccessDenied: + return err.Reason(), true + } +} + +func IsTimeoutError(err error) bool { + if strings.Contains(err.Error(), "timeout") || + errors.Is(err, context.DeadlineExceeded) { + return true + } + + return status.Code(UnwrapErr(err)) == codes.DeadlineExceeded +} diff --git a/internal/handler/download.go b/internal/handler/download.go index a7aee64..07fe3e9 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -17,7 +17,6 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -46,19 +45,20 @@ func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { h.byAttribute(c, h.receiveFile) } -func (h *Handler) search(ctx context.Context, cid *cid.ID, key, val string, op object.SearchMatchType) (pool.ResObjectSearch, error) { +func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) { filters := object.NewSearchFilters() filters.AddRootFilter() filters.AddFilter(key, val, op) - var prm pool.PrmObjectSearch - prm.SetContainerID(*cid) - prm.SetFilters(filters) - if btoken := bearerToken(ctx); btoken != nil { - prm.UseBearer(*btoken) + prm := PrmObjectSearch{ + PrmAuth: PrmAuth{ + BearerToken: bearerToken(ctx), + }, + Container: *cnrID, + Filters: filters, } - return h.pool.SearchObjects(ctx, prm) + return h.frostfs.SearchObjects(ctx, prm) } func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { @@ -153,18 +153,21 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { } func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error { - var prm pool.PrmObjectGet - prm.SetAddress(addr) - if btoken != nil { - prm.UseBearer(*btoken) + prm := PrmObjectRead{ + PrmAuth: PrmAuth{ + BearerToken: btoken, + }, + Address: addr, + WithHeader: true, + WithPayload: true, } - resGet, err := h.pool.GetObject(ctx, prm) + resGet, err := h.frostfs.ReadObject(ctx, prm) if err != nil { return fmt.Errorf("get FrostFS object: %v", err) } - objWriter, err := h.addObjectToZip(zipWriter, &resGet.Header) + objWriter, err := h.addObjectToZip(zipWriter, resGet.Head) if err != nil { return fmt.Errorf("zip create header: %v", err) } diff --git a/internal/handler/frostfs_mock.go b/internal/handler/frostfs_mock.go new file mode 100644 index 0000000..85ae874 --- /dev/null +++ b/internal/handler/frostfs_mock.go @@ -0,0 +1,260 @@ +package handler + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "fmt" + "io" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +type TestFrostFS struct { + objects map[string]*object.Object + containers map[string]*container.Container + accessList map[string]bool + key *keys.PrivateKey +} + +func NewTestFrostFS(key *keys.PrivateKey) *TestFrostFS { + return &TestFrostFS{ + objects: make(map[string]*object.Object), + containers: make(map[string]*container.Container), + accessList: make(map[string]bool), + key: key, + } +} + +func (t *TestFrostFS) ContainerID(name string) (*cid.ID, error) { + for id, cnr := range t.containers { + if container.Name(*cnr) == name { + var cnrID cid.ID + return &cnrID, cnrID.DecodeString(id) + } + } + return nil, fmt.Errorf("not found") +} + +func (t *TestFrostFS) SetContainer(cnrID cid.ID, cnr *container.Container) { + t.containers[cnrID.EncodeToString()] = cnr +} + +// AllowUserOperation grants access to object operations. +// Empty userID and objID means any user and object respectively. +func (t *TestFrostFS) AllowUserOperation(cnrID cid.ID, userID user.ID, op acl.Op, objID oid.ID) { + t.accessList[fmt.Sprintf("%s/%s/%s/%s", cnrID, userID, op, objID)] = true +} + +func (t *TestFrostFS) Container(_ context.Context, prm PrmContainer) (*container.Container, error) { + for k, v := range t.containers { + if k == prm.ContainerID.EncodeToString() { + return v, nil + } + } + + return nil, fmt.Errorf("container not found %s", prm.ContainerID) +} + +func (t *TestFrostFS) requestOwner(btoken *bearer.Token) user.ID { + if btoken != nil { + return bearer.ResolveIssuer(*btoken) + } + + var owner user.ID + user.IDFromKey(&owner, t.key.PrivateKey.PublicKey) + return owner +} + +func (t *TestFrostFS) ReadObject(_ context.Context, prm PrmObjectRead) (*ObjectPart, error) { + sAddr := prm.Address.EncodeToString() + + if obj, ok := t.objects[sAddr]; ok { + owner := t.requestOwner(prm.BearerToken) + + if !t.isAllowed(prm.Address.Container(), owner, acl.OpObjectGet, prm.Address.Object()) { + return nil, ErrAccessDenied + } + + payload := obj.Payload() + + if prm.PayloadRange[0]+prm.PayloadRange[1] > 0 { + off := prm.PayloadRange[0] + payload = payload[off : off+prm.PayloadRange[1]] + } + + return &ObjectPart{ + Head: obj, + Payload: io.NopCloser(bytes.NewReader(payload)), + }, nil + } + + return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, prm.Address) +} +func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID, error) { + b := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, b); err != nil { + return oid.ID{}, err + } + var id oid.ID + id.SetSHA256(sha256.Sum256(b)) + prm.Object.SetID(id) + + attrs := prm.Object.Attributes() + if prm.ClientCut { + a := object.NewAttribute() + a.SetKey("s3-client-cut") + a.SetValue("true") + attrs = append(attrs, *a) + } + + prm.Object.SetAttributes(attrs...) + + if prm.Payload != nil { + all, err := io.ReadAll(prm.Payload) + if err != nil { + return oid.ID{}, err + } + prm.Object.SetPayload(all) + prm.Object.SetPayloadSize(uint64(len(all))) + var hash checksum.Checksum + checksum.Calculate(&hash, checksum.SHA256, all) + prm.Object.SetPayloadChecksum(hash) + } + + cnrID, _ := prm.Object.ContainerID() + objID, _ := prm.Object.ID() + + owner := t.requestOwner(prm.BearerToken) + + if !t.isAllowed(cnrID, owner, acl.OpObjectPut, objID) { + return oid.ID{}, ErrAccessDenied + } + + addr := newAddress(cnrID, objID) + t.objects[addr.EncodeToString()] = prm.Object + return objID, nil +} + +type resObjectSearchMock struct { + res []oid.ID +} + +func (r *resObjectSearchMock) Read(buf []oid.ID) (int, error) { + for i := range buf { + if i > len(r.res)-1 { + return len(r.res), io.EOF + } + buf[i] = r.res[i] + } + + r.res = r.res[len(buf):] + + return len(buf), nil +} + +func (r *resObjectSearchMock) Iterate(f func(oid.ID) bool) error { + for _, id := range r.res { + if f(id) { + return nil + } + } + + return nil +} + +func (r *resObjectSearchMock) Close() {} + +func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) (ResObjectSearch, error) { + if !t.isAllowed(prm.Container, t.requestOwner(prm.BearerToken), acl.OpObjectSearch, oid.ID{}) { + return nil, ErrAccessDenied + } + + cidStr := prm.Container.EncodeToString() + var res []oid.ID + + if len(prm.Filters) == 1 { // match root filter + for k, v := range t.objects { + if strings.Contains(k, cidStr) { + id, _ := v.ID() + res = append(res, id) + } + } + return &resObjectSearchMock{res: res}, nil + } + + filter := prm.Filters[1] + if len(prm.Filters) != 2 || + filter.Operation() != object.MatchCommonPrefix && filter.Operation() != object.MatchStringEqual { + return nil, fmt.Errorf("usupported filters") + } + + for k, v := range t.objects { + if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) { + id, _ := v.ID() + res = append(res, id) + } + } + + return &resObjectSearchMock{res: res}, nil +} + +func isMatched(attributes []object.Attribute, filter object.SearchFilter) bool { + for _, attr := range attributes { + if attr.Key() == filter.Header() { + switch filter.Operation() { + case object.MatchStringEqual: + return attr.Value() == filter.Value() + case object.MatchCommonPrefix: + return strings.HasPrefix(attr.Value(), filter.Value()) + default: + return false + } + } + } + + return false +} + +func (t *TestFrostFS) GetEpochDurations(context.Context) (*utils.EpochDurations, error) { + return &utils.EpochDurations{ + CurrentEpoch: 10, + MsPerBlock: 1000, + BlockPerEpoch: 100, + }, nil +} + +func (t *TestFrostFS) isAllowed(cnrID cid.ID, userID user.ID, op acl.Op, objID oid.ID) bool { + keysToCheck := []string{ + fmt.Sprintf("%s/%s/%s/%s", cnrID, userID, op, objID), + fmt.Sprintf("%s/%s/%s/%s", cnrID, userID, op, oid.ID{}), + fmt.Sprintf("%s/%s/%s/%s", cnrID, user.ID{}, op, objID), + fmt.Sprintf("%s/%s/%s/%s", cnrID, user.ID{}, op, oid.ID{}), + } + + for _, key := range keysToCheck { + if t.accessList[key] { + return true + } + } + return false +} + +func newAddress(cnr cid.ID, obj oid.ID) oid.Address { + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(obj) + return addr +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index f88dff1..c87551e 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -12,16 +12,15 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -35,20 +34,125 @@ type Config interface { NamespaceHeader() string } +// PrmContainer groups parameters of FrostFS.Container operation. +type PrmContainer struct { + // Container identifier. + ContainerID cid.ID +} + +// PrmAuth groups authentication parameters for the FrostFS operation. +type PrmAuth struct { + // Bearer token to be used for the operation. Overlaps PrivateKey. Optional. + BearerToken *bearer.Token +} + +// PrmObjectRead groups parameters of FrostFS.ReadObject operation. +type PrmObjectRead struct { + // Authentication parameters. + PrmAuth + + // Address to read the object header from. + Address oid.Address + + // Flag to read object header. + WithHeader bool + + // Flag to read object payload. False overlaps payload range. + WithPayload bool + + // Offset-length range of the object payload to be read. + PayloadRange [2]uint64 +} + +// ObjectPart represents partially read FrostFS object. +type ObjectPart struct { + // Object header with optional in-memory payload part. + Head *object.Object + + // Object payload part encapsulated in io.Reader primitive. + // Returns ErrAccessDenied on read access violation. + Payload io.ReadCloser +} + +// PrmObjectCreate groups parameters of FrostFS.CreateObject operation. +type PrmObjectCreate struct { + // Authentication parameters. + PrmAuth + + Object *object.Object + + // Object payload encapsulated in io.Reader primitive. + Payload io.Reader + + // Enables client side object preparing. + ClientCut bool + + // Disables using Tillich-Zémor hash for payload. + WithoutHomomorphicHash bool + + // Sets max buffer size to read payload. + BufferMaxSize uint64 +} + +// PrmObjectSearch groups parameters of FrostFS.sear SearchObjects operation. +type PrmObjectSearch struct { + // Authentication parameters. + PrmAuth + + // Container to select the objects from. + Container cid.ID + + Filters object.SearchFilters +} + +type ResObjectSearch interface { + Read(buf []oid.ID) (int, error) + Iterate(f func(oid.ID) bool) error + Close() +} + +var ( + // ErrAccessDenied is returned from FrostFS in case of access violation. + ErrAccessDenied = errors.New("access denied") + // ErrGatewayTimeout is returned from FrostFS in case of timeout, deadline exceeded etc. + ErrGatewayTimeout = errors.New("gateway timeout") +) + +// FrostFS represents virtual connection to FrostFS network. +type FrostFS interface { + Container(context.Context, PrmContainer) (*container.Container, error) + ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error) + CreateObject(context.Context, PrmObjectCreate) (oid.ID, error) + SearchObjects(context.Context, PrmObjectSearch) (ResObjectSearch, error) + utils.EpochInfoFetcher +} + +type ContainerResolver interface { + Resolve(ctx context.Context, name string) (*cid.ID, error) +} + type Handler struct { log *zap.Logger - pool *pool.Pool + frostfs FrostFS ownerID *user.ID config Config - containerResolver *resolver.ContainerResolver + containerResolver ContainerResolver tree *tree.Tree cache *cache.BucketCache } -func New(params *utils.AppParams, config Config, tree *tree.Tree) *Handler { +type AppParams struct { + Logger *zap.Logger + FrostFS FrostFS + Owner *user.ID + Resolver ContainerResolver + Cache *cache.BucketCache +} + +func New(params *AppParams, config Config, tree *tree.Tree) *Handler { return &Handler{ log: params.Logger, - pool: params.Pool, + frostfs: params.FrostFS, ownerID: params.Owner, config: config, containerResolver: params.Resolver, @@ -235,8 +339,8 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log * } func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) { - prm := pool.PrmContainerGet{ContainerID: cnrID} - res, err := h.pool.GetContainer(ctx, prm) + prm := PrmContainer{ContainerID: cnrID} + res, err := h.frostfs.Container(ctx, prm) if err != nil { return nil, fmt.Errorf("get frostfs container '%s': %w", cnrID.String(), err) } @@ -246,12 +350,12 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket Name: cnrID.EncodeToString(), } - if domain := container.ReadDomain(res); domain.Name() != "" { + if domain := container.ReadDomain(*res); domain.Name() != "" { bktInfo.Name = domain.Name() bktInfo.Zone = domain.Zone() } - bktInfo.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(res) + bktInfo.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(*res) return bktInfo, err } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go new file mode 100644 index 0000000..ed67f88 --- /dev/null +++ b/internal/handler/handler_test.go @@ -0,0 +1,296 @@ +package handler + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "io" + "mime/multipart" + "net/http" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +type treeClientMock struct { +} + +func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) { + return nil, nil +} + +type configMock struct { +} + +func (c *configMock) DefaultTimestamp() bool { + return false +} + +func (c *configMock) ZipCompression() bool { + return false +} + +func (c *configMock) ClientCut() bool { + return false +} + +func (c *configMock) BufferMaxSizeForPut() uint64 { + return 0 +} + +func (c *configMock) NamespaceHeader() string { + return "" +} + +type handlerContext struct { + key *keys.PrivateKey + owner user.ID + + h *Handler + frostfs *TestFrostFS + tree *treeClientMock + cfg *configMock +} + +func (hc *handlerContext) Handler() *Handler { + return hc.h +} + +func prepareHandlerContext() (*handlerContext, error) { + logger, err := zap.NewDevelopment() + if err != nil { + return nil, err + } + + key, err := keys.NewPrivateKey() + if err != nil { + return nil, err + } + + var owner user.ID + user.IDFromKey(&owner, key.PrivateKey.PublicKey) + + testFrostFS := NewTestFrostFS(key) + + testResolver := &resolver.Resolver{Name: "test_resolver"} + testResolver.SetResolveFunc(func(_ context.Context, name string) (*cid.ID, error) { + return testFrostFS.ContainerID(name) + }) + + params := &AppParams{ + Logger: logger, + FrostFS: testFrostFS, + Owner: &owner, + Resolver: testResolver, + Cache: cache.NewBucketCache(&cache.Config{ + Size: 1, + Lifetime: 1, + Logger: logger, + }), + } + + treeMock := &treeClientMock{} + cfgMock := &configMock{} + + handler := New(params, cfgMock, tree.NewTree(treeMock)) + + return &handlerContext{ + key: key, + owner: owner, + h: handler, + frostfs: testFrostFS, + tree: treeMock, + cfg: cfgMock, + }, nil +} + +func (hc *handlerContext) prepareContainer(name string, basicACL acl.Basic) (cid.ID, *container.Container, error) { + var pp netmap.PlacementPolicy + err := pp.DecodeString("REP 1") + if err != nil { + return cid.ID{}, nil, err + } + + var cnr container.Container + cnr.Init() + cnr.SetOwner(hc.owner) + cnr.SetPlacementPolicy(pp) + cnr.SetBasicACL(basicACL) + + var domain container.Domain + domain.SetName(name) + container.WriteDomain(&cnr, domain) + container.SetName(&cnr, name) + container.SetCreationTime(&cnr, time.Now()) + + cnrID := cidtest.ID() + + for op := acl.OpObjectGet; op < acl.OpObjectHash; op++ { + hc.frostfs.AllowUserOperation(cnrID, hc.owner, op, oid.ID{}) + if basicACL.IsOpAllowed(op, acl.RoleOthers) { + hc.frostfs.AllowUserOperation(cnrID, user.ID{}, op, oid.ID{}) + } + } + + return cnrID, &cnr, nil +} + +func TestBasic(t *testing.T) { + hc, err := prepareHandlerContext() + require.NoError(t, err) + + bktName := "bucket" + cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended) + require.NoError(t, err) + hc.frostfs.SetContainer(cnrID, cnr) + + ctx := context.Background() + ctx = middleware.SetNamespace(ctx, "") + + content := "hello" + r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content) + require.NoError(t, err) + + hc.Handler().Upload(r) + require.Equal(t, r.Response.StatusCode(), http.StatusOK) + + var putRes putResponse + err = json.Unmarshal(r.Response.Body(), &putRes) + require.NoError(t, err) + + obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] + attr := object.NewAttribute() + attr.SetKey(object.AttributeFilePath) + attr.SetValue(objFileName) + obj.SetAttributes(append(obj.Attributes(), *attr)...) + + t.Run("get", func(t *testing.T) { + r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID) + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, content, string(r.Response.Body())) + }) + + t.Run("head", func(t *testing.T) { + r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID) + hc.Handler().HeadByAddressOrBucketName(r) + require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) + require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + }) + + t.Run("get by attribute", func(t *testing.T) { + r = prepareGetByAttributeRequest(ctx, bktName, keyAttr, valAttr) + hc.Handler().DownloadByAttribute(r) + require.Equal(t, content, string(r.Response.Body())) + }) + + t.Run("head by attribute", func(t *testing.T) { + r = prepareGetByAttributeRequest(ctx, bktName, keyAttr, valAttr) + hc.Handler().HeadByAttribute(r) + require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) + require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + }) + + t.Run("zip", func(t *testing.T) { + r = prepareGetZipped(ctx, bktName, "") + hc.Handler().DownloadZipped(r) + + readerAt := bytes.NewReader(r.Response.Body()) + zipReader, err := zip.NewReader(readerAt, int64(len(r.Response.Body()))) + require.NoError(t, err) + require.Len(t, zipReader.File, 1) + require.Equal(t, objFileName, zipReader.File[0].Name) + f, err := zipReader.File[0].Open() + require.NoError(t, err) + defer func() { + inErr := f.Close() + require.NoError(t, inErr) + }() + data, err := io.ReadAll(f) + require.NoError(t, err) + require.Equal(t, content, string(data)) + }) +} + +func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", bucket) + return r, fillMultipartBody(r, content) +} + +func prepareGetRequest(ctx context.Context, bucket, objID string) *fasthttp.RequestCtx { + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", bucket) + r.SetUserValue("oid", objID) + return r +} + +func prepareGetByAttributeRequest(ctx context.Context, bucket, attrKey, attrVal string) *fasthttp.RequestCtx { + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", bucket) + r.SetUserValue("attr_key", attrKey) + r.SetUserValue("attr_val", attrVal) + return r +} + +func prepareGetZipped(ctx context.Context, bucket, prefix string) *fasthttp.RequestCtx { + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", bucket) + r.SetUserValue("prefix", prefix) + return r +} + +const ( + keyAttr = "User-Attribute" + valAttr = "user value" + objFileName = "newFile.txt" +) + +func fillMultipartBody(r *fasthttp.RequestCtx, content string) error { + attributes := map[string]string{ + object.AttributeFileName: objFileName, + keyAttr: valAttr, + } + + var buff bytes.Buffer + w := multipart.NewWriter(&buff) + fw, err := w.CreateFormFile("file", attributes[object.AttributeFileName]) + if err != nil { + return err + } + + if _, err = io.Copy(fw, bytes.NewBufferString(content)); err != nil { + return err + } + + if err = w.Close(); err != nil { + return err + } + + r.Request.SetBodyStream(&buff, buff.Len()) + r.Request.Header.Set("Content-Type", w.FormDataContentType()) + r.Request.Header.Set("X-Attribute-"+keyAttr, valAttr) + + return nil +} diff --git a/internal/handler/head.go b/internal/handler/head.go index 9418567..2a17f64 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -11,7 +11,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -30,21 +29,23 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid btoken := bearerToken(ctx) - var prm pool.PrmObjectHead - prm.SetAddress(objectAddress) - if btoken != nil { - prm.UseBearer(*btoken) + prm := PrmObjectRead{ + PrmAuth: PrmAuth{ + BearerToken: btoken, + }, + Address: objectAddress, + WithHeader: true, } - obj, err := h.pool.HeadObject(ctx, prm) + obj, err := h.frostfs.ReadObject(ctx, prm) if err != nil { req.handleFrostFSErr(err, start) return } - req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10)) + req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.Head.PayloadSize(), 10)) var contentType string - for _, attr := range obj.Attributes() { + for _, attr := range obj.Head.Attributes() { key := attr.Key() val := attr.Value() if !isValidToken(key) || !isValidValue(val) { @@ -70,22 +71,24 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid } } - idsToResponse(&req.Response, &obj) + idsToResponse(&req.Response, obj.Head) if len(contentType) == 0 { - contentType, _, err = readContentType(obj.PayloadSize(), func(sz uint64) (io.Reader, error) { - var prmRange pool.PrmObjectRange - prmRange.SetAddress(objectAddress) - prmRange.SetLength(sz) - if btoken != nil { - prmRange.UseBearer(*btoken) + contentType, _, err = readContentType(obj.Head.PayloadSize(), func(sz uint64) (io.Reader, error) { + prmRange := PrmObjectRead{ + PrmAuth: PrmAuth{ + BearerToken: btoken, + }, + Address: objectAddress, + WithPayload: true, + PayloadRange: [2]uint64{0, sz}, } - resObj, err := h.pool.ObjectRange(ctx, prmRange) + resObj, err := h.frostfs.ReadObject(ctx, prmRange) if err != nil { return nil, err } - return &resObj, nil + return resObj.Payload, nil }) if err != nil && err != io.EOF { req.handleFrostFSErr(err, start) diff --git a/internal/handler/reader.go b/internal/handler/reader.go index 76801f7..b48ac6d 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -14,7 +14,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -56,13 +55,16 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi filename string ) - var prm pool.PrmObjectGet - prm.SetAddress(objectAddress) - if btoken := bearerToken(ctx); btoken != nil { - prm.UseBearer(*btoken) + prm := PrmObjectRead{ + PrmAuth: PrmAuth{ + BearerToken: bearerToken(ctx), + }, + Address: objectAddress, + WithHeader: true, + WithPayload: true, } - rObj, err := h.pool.GetObject(ctx, prm) + rObj, err := h.frostfs.ReadObject(ctx, prm) if err != nil { req.handleFrostFSErr(err, start) return @@ -74,11 +76,11 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi dis = "attachment" } - payloadSize := rObj.Header.PayloadSize() + payloadSize := rObj.Head.PayloadSize() req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) var contentType string - for _, attr := range rObj.Header.Attributes() { + for _, attr := range rObj.Head.Attributes() { key := attr.Key() val := attr.Value() if !isValidToken(key) || !isValidValue(val) { @@ -107,7 +109,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi } } - idsToResponse(&req.Response, &rObj.Header) + idsToResponse(&req.Response, rObj.Head) if len(contentType) == 0 { // determine the Content-Type from the payload head diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 1b18755..cea2250 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -15,7 +15,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -98,7 +97,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { } } - if err = utils.PrepareExpirationHeader(req, h.pool, filtered, now); err != nil { + if err = utils.PrepareExpirationHeader(req, h.frostfs, filtered, now); err != nil { log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) response.Error(req, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) return @@ -132,19 +131,18 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { obj.SetOwnerID(*h.ownerID) obj.SetAttributes(attributes...) - var prm pool.PrmObjectPut - prm.SetHeader(*obj) - prm.SetPayload(file) - prm.SetClientCut(h.config.ClientCut()) - prm.SetBufferMaxSize(h.config.BufferMaxSizeForPut()) - prm.WithoutHomomorphicHash(bktInfo.HomomorphicHashDisabled) - - bt := h.fetchBearerToken(ctx) - if bt != nil { - prm.UseBearer(*bt) + prm := PrmObjectCreate{ + PrmAuth: PrmAuth{ + BearerToken: h.fetchBearerToken(ctx), + }, + Object: obj, + Payload: file, + ClientCut: h.config.ClientCut(), + WithoutHomomorphicHash: bktInfo.HomomorphicHashDisabled, + BufferMaxSize: h.config.BufferMaxSizeForPut(), } - if idObj, err = h.pool.PutObject(ctx, prm); err != nil { + if idObj, err = h.frostfs.CreateObject(ctx, prm); err != nil { h.handlePutFrostFSErr(req, err) return } diff --git a/utils/attributes.go b/utils/attributes.go index cfa3e3a..4d277a9 100644 --- a/utils/attributes.go +++ b/utils/attributes.go @@ -11,10 +11,18 @@ import ( "time" "unicode" "unicode/utf8" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" ) +type EpochDurations struct { + CurrentEpoch uint64 + MsPerBlock int64 + BlockPerEpoch uint64 +} + +type EpochInfoFetcher interface { + GetEpochDurations(context.Context) (*EpochDurations, error) +} + const ( UserAttributeHeaderPrefix = "X-Attribute-" ) @@ -151,7 +159,7 @@ func title(str string) string { return string(r0) + str[size:] } -func PrepareExpirationHeader(ctx context.Context, p *pool.Pool, headers map[string]string, now time.Time) error { +func PrepareExpirationHeader(ctx context.Context, epochFetcher EpochInfoFetcher, headers map[string]string, now time.Time) error { formatsNum := 0 index := -1 for i, transformer := range transformers { @@ -165,7 +173,7 @@ func PrepareExpirationHeader(ctx context.Context, p *pool.Pool, headers map[stri case 0: return nil case 1: - epochDuration, err := GetEpochDurations(ctx, p) + epochDuration, err := epochFetcher.GetEpochDurations(ctx) if err != nil { return fmt.Errorf("couldn't get epoch durations from network info: %w", err) } diff --git a/utils/params.go b/utils/params.go deleted file mode 100644 index f27ff71..0000000 --- a/utils/params.go +++ /dev/null @@ -1,17 +0,0 @@ -package utils - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "go.uber.org/zap" -) - -type AppParams struct { - Logger *zap.Logger - Pool *pool.Pool - Owner *user.ID - Resolver *resolver.ContainerResolver - Cache *cache.BucketCache -} diff --git a/utils/util.go b/utils/util.go index a328769..d513817 100644 --- a/utils/util.go +++ b/utils/util.go @@ -2,36 +2,10 @@ package utils import ( "context" - "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/valyala/fasthttp" ) -type EpochDurations struct { - CurrentEpoch uint64 - MsPerBlock int64 - BlockPerEpoch uint64 -} - -func GetEpochDurations(ctx context.Context, p *pool.Pool) (*EpochDurations, error) { - networkInfo, err := p.NetworkInfo(ctx) - if err != nil { - return nil, err - } - - res := &EpochDurations{ - CurrentEpoch: networkInfo.CurrentEpoch(), - MsPerBlock: networkInfo.MsPerBlock(), - BlockPerEpoch: networkInfo.EpochDuration(), - } - - if res.BlockPerEpoch == 0 { - return nil, fmt.Errorf("EpochDuration is empty") - } - return res, nil -} - // SetContextToRequest adds new context to fasthttp request. func SetContextToRequest(ctx context.Context, c *fasthttp.RequestCtx) { c.SetUserValue("context", ctx) From 27478995b5889cc79bdd5c477f4d97c02d5e4509 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 24 Jun 2024 16:54:55 +0300 Subject: [PATCH 454/548] [#118] Replace ACLs with polices in readme Signed-off-by: Alex Vanin --- README.md | 87 ++++++++++++++++++++++--------------------------------- 1 file changed, 35 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 6e19d31..aa982db 100644 --- a/README.md +++ b/README.md @@ -466,13 +466,13 @@ You can always upload files to public containers (open for anyone to put objects into), but for restricted containers you need to explicitly allow PUT operations for a request signed with your HTTP Gateway keys. -If your don't want to manage gateway's secret keys and adjust eACL rules when +If you don't want to manage gateway's secret keys and adjust policies when gateway configuration changes (new gate, key rotation, etc) or you plan to use public services, there is an option to let your application backend (or you) to -issue Bearer Tokens ans pass them from the client via gate down to FrostFS level +issue Bearer Tokens and pass them from the client via gate down to FrostFS level to grant access. -FrostFS Bearer Token basically is a container owner-signed ACL data (refer to FrostFS +FrostFS Bearer Token basically is a container owner-signed policy (refer to FrostFS documentation for more details). There are two options to pass them to gateway: * "Authorization" header with "Bearer" type and base64-encoded token in credentials field @@ -482,33 +482,31 @@ For example, you have a mobile application frontend with a backend part storing data in FrostFS. When a user authorizes in the mobile app, the backend issues a FrostFS Bearer token and provides it to the frontend. Then, the mobile app may generate some data and upload it via any available FrostFS HTTP Gateway by adding -the corresponding header to the upload request. Accessing the ACL protected data +the corresponding header to the upload request. Accessing policy protected data works the same way. ##### Example -In order to generate a bearer token, you need to have wallet (which will be used to sign the token) and -the address of the sender who will do the request to FrostFS (in our case, it's a gateway wallet address). +In order to generate a bearer token, you need to have wallet (which will be used to sign the token) -Suppose we have: -* **NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3** (token owner (gateway address)) +1. Suppose you have a container with private policy for wallet key -Firstly, we need to encode the container id and the sender address to base64 (now it's base58). -So use **base58** and **base64** utils. - -1. Encoding token owner id: ``` -$ echo 'NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3' | base58 --decode | base64 -# output: NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg== +$ frostfs-cli container create -r --wallet -policy --basic-acl 0 --await +CID: 9dfzyvq82JnFqp5svxcREf2iy6XNuifYcJPusEDnGK9Z + +$ frostfs-cli ape-manager add -r --wallet \ + --target-type container --target-name 9dfzyvq82JnFqp5svxcREf2iy6XNuifYcJPusEDnGK9Z \ + --rule "allow Object.* RequestCondition:"\$Actor:publicKey"=03b09baabff3f6107c7e9acb8721a6fc5618d45b50247a314d82e548702cce8cd5 *" \ + --chain-id ``` -2. Form a Bearer token (10000 is lifetime expiration in epoch) and save it to **bearer.json**: + +2. Form a Bearer token (10000 is lifetime expiration in epoch) to impersonate + HTTP Gateway request as wallet signed request and save it to **bearer.json**: ``` { "body": { "allowImpersonate": true, - "ownerID": { - "value": "NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg==" - }, "lifetime": { "exp": "10000", "nbf": "0", @@ -521,7 +519,7 @@ $ echo 'NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3' | base58 --decode | base64 3. Sign it with the wallet: ``` -$ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w ./wallet.json +$ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w ``` 4. Encode to base64 to use in header: @@ -542,47 +540,32 @@ $ curl -F 'file=@cat.jpeg;filename=cat.jpeg' -H "Authorization: Bearer Ck4KKgoEC # } ``` -##### Note -For the token to work correctly, you need to create a container with a basic ACL that: -1. Allow PUT operation to others -2. Doesn't set "final" bit +##### Note: Bearer Token owner + +You can specify exact key who can use Bearer Token (gateway wallet address). +To do this, encode wallet address in base64 format -For example: ``` -$ frostfs-cli -w ./wallet.json --basic-acl 0x0FFFCFFF -r 192.168.130.72:8080 container create --policy "REP 3" --await +$ echo 'NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3' | base58 --decode | base64 +# output: NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg== ``` -To deny access to a container without a token, set the eACL rules: -``` -$ frostfs-cli -w ./wallet.json -r 192.168.130.72:8080 container set-eacl --table eacl.json --await --cid BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K -``` - -File **eacl.json**: +Then specify this value in Bearer Token Json ``` { - "version": { - "major": 0, - "minor": 0 - }, - "containerID": { - "value": "mRnZWzewzxjzIPa7Fqlfqdl3TM1KpJ0YnsXsEhafJJg=" - }, - "records": [ - { - "operation": "PUT", - "action": "DENY", - "filters": [], - "targets": [ - { - "role": "OTHERS", - "keys": [] - } - ] - } - ] -} + "body": { + "ownerID": { + "value": "NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg==" + }, + ... ``` +##### Note: Policy override + +Instead of impersonation, you can define the set of policies that will be applied +to the request sender. This allows to restrict access to specific operation and +specific objects without giving full impersonation control to the token user. + ### Metrics and Pprof If enabled, Prometheus metrics are available at `localhost:8084` endpoint From 0f22ca43c126df8deb75145fea115176c78d49fd Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 25 Jun 2024 15:31:46 +0300 Subject: [PATCH 455/548] [#117] Fix FrostFS interface usage HTTP Gateway expects io.Reader to work with payload, however `WithPayload` flag reads whole payload into header object. Signed-off-by: Alex Vanin --- internal/frostfs/frostfs.go | 46 ++++++++---------------------------- internal/handler/download.go | 4 +--- internal/handler/handler.go | 6 ----- internal/handler/head.go | 4 +--- internal/handler/reader.go | 4 +--- 5 files changed, 13 insertions(+), 51 deletions(-) diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go index dde560b..fc41420 100644 --- a/internal/frostfs/frostfs.go +++ b/internal/frostfs/frostfs.go @@ -85,43 +85,16 @@ func (x *FrostFS) ReadObject(ctx context.Context, prm handler.PrmObjectRead) (*h prmGet.UseBearer(*prm.BearerToken) } - if prm.WithHeader { - if prm.WithPayload { - res, err := x.pool.GetObject(ctx, prmGet) - if err != nil { - return nil, handleObjectError("init full object reading via connection pool", err) - } + // The code below must be reworked. It was copied from frostfs-s3-gw + // to create similar mocks for unit and fuzzing tests. + // + // However, this code was changed due to specific of expected responses + // from HTTP gateway. HTTP Gateway requires two types of responses: + // * payload as io.Reader + HEAD request + // * only payload as io.Reader + // Therefore all unused params were deleted and code was simplified. - defer res.Payload.Close() - - payload, err := io.ReadAll(res.Payload) - if err != nil { - return nil, handleObjectError("read full object payload", err) - } - - res.Header.SetPayload(payload) - - return &handler.ObjectPart{ - Head: &res.Header, - }, nil - } - - var prmHead pool.PrmObjectHead - prmHead.SetAddress(prm.Address) - - if prm.BearerToken != nil { - prmHead.UseBearer(*prm.BearerToken) - } - - hdr, err := x.pool.HeadObject(ctx, prmHead) - if err != nil { - return nil, handleObjectError("read object header via connection pool", err) - } - - return &handler.ObjectPart{ - Head: &hdr, - }, nil - } else if prm.PayloadRange[0]+prm.PayloadRange[1] == 0 { + if prm.PayloadRange[0]+prm.PayloadRange[1] == 0 { res, err := x.pool.GetObject(ctx, prmGet) if err != nil { return nil, handleObjectError("init full payload range reading via connection pool", err) @@ -129,6 +102,7 @@ func (x *FrostFS) ReadObject(ctx context.Context, prm handler.PrmObjectRead) (*h return &handler.ObjectPart{ Payload: res.Payload, + Head: &res.Header, }, nil } diff --git a/internal/handler/download.go b/internal/handler/download.go index 07fe3e9..480254c 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -157,9 +157,7 @@ func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid PrmAuth: PrmAuth{ BearerToken: btoken, }, - Address: addr, - WithHeader: true, - WithPayload: true, + Address: addr, } resGet, err := h.frostfs.ReadObject(ctx, prm) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index c87551e..197649e 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -54,12 +54,6 @@ type PrmObjectRead struct { // Address to read the object header from. Address oid.Address - // Flag to read object header. - WithHeader bool - - // Flag to read object payload. False overlaps payload range. - WithPayload bool - // Offset-length range of the object payload to be read. PayloadRange [2]uint64 } diff --git a/internal/handler/head.go b/internal/handler/head.go index 2a17f64..96d1f49 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -33,8 +33,7 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid PrmAuth: PrmAuth{ BearerToken: btoken, }, - Address: objectAddress, - WithHeader: true, + Address: objectAddress, } obj, err := h.frostfs.ReadObject(ctx, prm) @@ -80,7 +79,6 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid BearerToken: btoken, }, Address: objectAddress, - WithPayload: true, PayloadRange: [2]uint64{0, sz}, } diff --git a/internal/handler/reader.go b/internal/handler/reader.go index b48ac6d..81694bc 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -59,9 +59,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi PrmAuth: PrmAuth{ BearerToken: bearerToken(ctx), }, - Address: objectAddress, - WithHeader: true, - WithPayload: true, + Address: objectAddress, } rObj, err := h.frostfs.ReadObject(ctx, prm) From 1737f1d95fdc918ef1d7c8a91075fa9f978c37f6 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 25 Jun 2024 16:12:30 +0300 Subject: [PATCH 456/548] [#117] Update tests Signed-off-by: Alex Vanin --- cmd/http-gw/integration_test.go | 1 + internal/handler/frostfs_mock.go | 1 + 2 files changed, 2 insertions(+) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index cae40a5..e888ed6 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -54,6 +54,7 @@ func TestIntegration(t *testing.T) { versions := []string{ "1.2.7", "1.3.0", + "1.5.0", } key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) diff --git a/internal/handler/frostfs_mock.go b/internal/handler/frostfs_mock.go index 85ae874..eb59fb6 100644 --- a/internal/handler/frostfs_mock.go +++ b/internal/handler/frostfs_mock.go @@ -93,6 +93,7 @@ func (t *TestFrostFS) ReadObject(_ context.Context, prm PrmObjectRead) (*ObjectP if prm.PayloadRange[0]+prm.PayloadRange[1] > 0 { off := prm.PayloadRange[0] payload = payload[off : off+prm.PayloadRange[1]] + obj = nil // GetRange does not return header values } return &ObjectPart{ From d9cbd302b18be699751bbc40f953be8a0abe41ad Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 25 Jun 2024 19:01:58 +0300 Subject: [PATCH 457/548] [#121] Add canonicalizer Some headers might be passed in non-canonical way by proxy servers, such as 'Authorization' header. Server does not normalize headers, so we can get custom object attributes. Therefore, app has to normalize all non object attribute headers by itself. Signed-off-by: Alex Vanin --- cmd/http-gw/app.go | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 1d12122..3d02faa 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "errors" "fmt" @@ -534,15 +535,15 @@ func (a *app) configureRouter(handler *handler.Handler) { response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.Upload))))) + r.POST("/upload/{cid}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.Upload)))))) a.log.Info(logs.AddedPathUploadCid) - r.GET("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadByAddressOrBucketName))))) - r.HEAD("/get/{cid}/{oid:*}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.HeadByAddressOrBucketName))))) + r.GET("/get/{cid}/{oid:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadByAddressOrBucketName)))))) + r.HEAD("/get/{cid}/{oid:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.HeadByAddressOrBucketName)))))) a.log.Info(logs.AddedPathGetCidOid) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadByAttribute))))) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.HeadByAttribute))))) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadByAttribute)))))) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.HeadByAttribute)))))) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.logger(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadZipped))))) + r.GET("/zip/{cid}/{prefix:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadZipped)))))) a.log.Info(logs.AddedPathZipCidPrefix) a.webServer.Handler = r.Handler @@ -559,6 +560,38 @@ func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { } } +func (a *app) canonicalizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { + return func(req *fasthttp.RequestCtx) { + // regardless of DisableHeaderNamesNormalizing setting, some headers + // MUST be normalized in order to process execution. They are normalized + // here. + + toAddKeys := make([][]byte, 0, 10) + toAddValues := make([][]byte, 0, 10) + prefix := []byte(utils.UserAttributeHeaderPrefix) + + req.Request.Header.VisitAll(func(k, v []byte) { + if bytes.HasPrefix(k, prefix) { + return + } + toAddKeys = append(toAddKeys, k) + toAddValues = append(toAddValues, v) + }) + + // this is safe to do after all headers were read into header structure + req.Request.Header.EnableNormalizing() + + for i := range toAddKeys { + req.Request.Header.SetBytesKV(toAddKeys[i], toAddValues[i]) + } + + // return normalization setting back + req.Request.Header.DisableNormalizing() + + h(req) + } +} + func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { appCtx, err := tokens.StoreBearerTokenAppCtx(a.ctx, req) From 16545bd3b0149367417e6b329d10c377af31d2f8 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Fri, 5 Jul 2024 13:24:54 +0300 Subject: [PATCH 458/548] [#124] Update SDK version Signed-off-by: Marina Biryukova --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a34a092..f9a04b1 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 github.com/fasthttp/router v1.4.1 diff --git a/go.sum b/go.sum index 0cde076..5baaf63 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f h1:vBLC1OSGMSn7lRJv/p1of0veifuBdZdztVrF9Vn+UFk= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4 h1:izmHYpkz7cPr2Zpudxxh0wvrtAIxYywEG+uraghVSlo= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 418767c8ec57e00a238472924751b6d2a2f2c1a8 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 19 Jul 2024 17:48:01 +0300 Subject: [PATCH 459/548] [#129] Update FrostFS API and remove unused code Signed-off-by: Alex Vanin --- go.mod | 49 ++-- go.sum | 278 ++++++---------------- internal/frostfs/services/pool_wrapper.go | 24 -- 3 files changed, 99 insertions(+), 252 deletions(-) diff --git a/go.mod b/go.mod index f9a04b1..493a003 100644 --- a/go.mod +++ b/go.mod @@ -3,31 +3,31 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.21 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 github.com/fasthttp/router v1.4.1 - github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc - github.com/prometheus/client_golang v1.15.1 - github.com/prometheus/client_model v0.3.0 + github.com/nspcc-dev/neo-go v0.106.2 + github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_model v0.5.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/ssgreg/journald v1.0.0 - github.com/stretchr/testify v1.8.3 + github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.13.0 github.com/valyala/fasthttp v1.34.0 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 - go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc + go.uber.org/zap v1.27.0 + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/net v0.23.0 - google.golang.org/grpc v1.61.1 + google.golang.org/grpc v1.62.0 ) require ( - git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb // indirect + git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e // indirect git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect @@ -42,7 +42,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/containerd v1.6.2 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/distribution v2.8.1+incompatible // indirect @@ -55,33 +55,32 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.0 // indirect - github.com/gorilla/websocket v1.5.0 // indirect + github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect - github.com/hashicorp/golang-lru v0.6.0 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/klauspost/compress v1.16.4 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/sys/mount v0.3.2 // indirect github.com/moby/sys/mountinfo v0.6.1 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect - github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect - github.com/nspcc-dev/rfc6979 v0.2.0 // indirect + github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect + github.com/nspcc-dev/rfc6979 v0.2.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.1.1 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect github.com/sirupsen/logrus v1.8.1 // indirect @@ -93,6 +92,7 @@ require ( github.com/twmb/murmur3 v1.1.8 // indirect github.com/urfave/cli v1.22.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect + go.etcd.io/bbolt v1.3.9 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect @@ -101,17 +101,16 @@ require ( go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/otel/sdk v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sync v0.5.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 5baaf63..73b2798 100644 --- a/go.sum +++ b/go.sum @@ -37,16 +37,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 h1:H5GvrVlowIMWfzqQkhY0p0myooJxQ1sMRVSFfXawwWg= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 h1:XxvwQKJT/f16qS3df5PBQPRYKkhy0/A7zH6644QpKD0= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4 h1:izmHYpkz7cPr2Zpudxxh0wvrtAIxYywEG+uraghVSlo= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 h1:MV/vKJWLQT34RRbXYvkNKFYGNjL5bRNuCQMXkbC7fLI= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36/go.mod h1:vluJ/+yQMcq8ZIZZSA7Te+JKClr0lgtRErjICvb8wto= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= @@ -71,10 +71,6 @@ github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1:MJCkWUBhi9pn/CrYO1Q3P687y2KeahrOPS9BD9LDGb0= -github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= -github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= -github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= @@ -105,31 +101,20 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= -github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= -github.com/abiosoft/ishell/v2 v2.0.2/go.mod h1:E4oTCXfo6QjoCart0QYa5m9w4S+deXs/P/9jA77A9Bs= -github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= -github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= -github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -138,24 +123,14 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= +github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= @@ -166,9 +141,7 @@ github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqy github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= @@ -193,6 +166,10 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb h1:f0BMgIjhZy4lSRHCXFbQst85f5agZAjtDMixQqBWNpc= +github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= @@ -303,8 +280,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= @@ -315,13 +292,11 @@ github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1S github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -363,17 +338,13 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fasthttp/router v1.4.1 h1:3xPUO+hy/HAkgGDSd5sX5w18cyGDIFbC7vip8KwPDk8= github.com/fasthttp/router v1.4.1/go.mod h1:4P0Kq4C882tA2evBKDW7De7hGfWmvV8FN+zqt8Lu49Q= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -390,12 +361,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -414,13 +381,11 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= @@ -438,8 +403,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -454,7 +419,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -474,12 +438,9 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -494,7 +455,6 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= @@ -535,8 +495,8 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -553,14 +513,12 @@ github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1: github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= -github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= +github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -571,31 +529,22 @@ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= @@ -610,7 +559,6 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -629,21 +577,14 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= @@ -652,6 +593,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= github.com/moby/sys/mount v0.3.2 h1:uq/CiGDZPvr+c85RYHtKIUORFbmavBUyWH3E1NEyjqI= @@ -669,74 +612,49 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= -github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= -github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= -github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= -github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= -github.com/nspcc-dev/dbft v0.0.0-20220902113116-58a5e763e647/go.mod h1:g9xisXmX9NP9MjioaTe862n9SlZTrP+6PVUWLBYOr98= -github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= -github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 h1:n4ZaFCKt1pQJd7PXoMJabZWK9ejjbLOVrkl/lOUmshg= -github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= -github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= -github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= -github.com/nspcc-dev/neo-go v0.99.4/go.mod h1:mKTolfRUfKjFso5HPvGSQtUZc70n0VKBMs16eGuC5gA= -github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc h1:fySIWvUQsitK5e5qYIHnTDCXuPpwzz89SEUEIyY11sg= -github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc/go.mod h1:s9QhjMC784MWqTURovMbyYduIJc86mnCruxcMiAebpc= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce h1:vLGuUNDkmQrWMa4rr4vTd1u8ULqejWxVmNz1L7ocTEI= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= -github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= -github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= -github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= -github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= +github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk= +github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc= +github.com/nspcc-dev/neo-go v0.106.2 h1:KXSJ2J5Oacc7LrX3r4jvnC8ihKqHs5NB21q4f2S3r9o= +github.com/nspcc-dev/neo-go v0.106.2/go.mod h1:Ojwfx3/lv0VTeEHMpQ17g0wTnXcCSoFQVq5GEeCZmGo= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d h1:Vcb7YkZuUSSIC+WF/xV3UDfHbAxZgyT2zGleJP3Ig5k= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d/go.mod h1:/vrbWSHc7YS1KSYhVOyyeucXW/e+1DkVBOgnBEXUCeY= +github.com/nspcc-dev/rfc6979 v0.2.1 h1:8wWxkamHWFmO790GsewSoKUSJjVnL1fmdRpokU/RgRM= +github.com/nspcc-dev/rfc6979 v0.2.1/go.mod h1:Tk7h5kyUWkhjyO3zUgFFhy1v2vQv3BvQEntakdtqrWc= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -772,7 +690,6 @@ github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrap github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -787,32 +704,24 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -824,17 +733,14 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -859,7 +765,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= @@ -901,15 +806,13 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= @@ -917,7 +820,6 @@ github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBN github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= @@ -933,8 +835,6 @@ github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfY github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= -github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= @@ -953,17 +853,15 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= -github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= +go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -995,23 +893,15 @@ go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJP go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1021,15 +911,12 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= @@ -1044,8 +931,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1070,8 +957,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1119,12 +1006,9 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= @@ -1137,9 +1021,7 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1151,17 +1033,14 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1178,14 +1057,11 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1213,7 +1089,6 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1240,23 +1115,19 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= @@ -1280,7 +1151,6 @@ golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1302,7 +1172,6 @@ golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDq golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1333,7 +1202,6 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1343,11 +1211,14 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1418,12 +1289,12 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= -google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= -google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= -google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 h1:x9PwdEgd11LgK+orcck69WVRo7DezSO4VUMPI4xpc8A= +google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go.mod h1:rbHMSEDyoYX62nRVLOCc4Qt1HbsdytAYoVwgjiOhF3I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c h1:NUsgEN92SQQqzfA+YtqYNqYmB3DMMYLlIwUZAQFVFbo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1447,10 +1318,9 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY= -google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= +google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1464,10 +1334,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1489,6 +1357,7 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1498,6 +1367,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1549,6 +1419,8 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/internal/frostfs/services/pool_wrapper.go b/internal/frostfs/services/pool_wrapper.go index 039d575..f7b0a26 100644 --- a/internal/frostfs/services/pool_wrapper.go +++ b/internal/frostfs/services/pool_wrapper.go @@ -35,30 +35,6 @@ func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { return res } -type GetSubTreeResponseBodyWrapper struct { - response *grpcService.GetSubTreeResponse_Body -} - -func (n GetSubTreeResponseBodyWrapper) GetNodeID() uint64 { - return n.response.GetNodeId() -} - -func (n GetSubTreeResponseBodyWrapper) GetParentID() uint64 { - return n.response.GetParentId() -} - -func (n GetSubTreeResponseBodyWrapper) GetTimestamp() uint64 { - return n.response.GetTimestamp() -} - -func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta { - res := make([]tree.Meta, len(n.response.Meta)) - for i, value := range n.response.Meta { - res[i] = value - } - return res -} - type PoolWrapper struct { p *treepool.Pool } From 9e2d1208cb20a2993de526061f859e2dc0ce3533 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 19 Jul 2024 17:55:34 +0300 Subject: [PATCH 460/548] [#129] Remove resolver duplicate Signed-off-by: Alex Vanin --- cmd/http-gw/app.go | 2 +- resolver/frostfs.go | 35 ----------------------------------- 2 files changed, 1 insertion(+), 36 deletions(-) delete mode 100644 resolver/frostfs.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 3d02faa..561598f 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -223,7 +223,7 @@ func (a *app) initResolver() { func (a *app) getResolverConfig() ([]string, *resolver.Config) { resolveCfg := &resolver.Config{ - FrostFS: resolver.NewFrostFSResolver(a.pool), + FrostFS: frostfs.NewResolverFrostFS(a.pool), RPCAddress: a.cfg.GetString(cfgRPCEndpoint), Settings: a.settings, } diff --git a/resolver/frostfs.go b/resolver/frostfs.go deleted file mode 100644 index aa7a751..0000000 --- a/resolver/frostfs.go +++ /dev/null @@ -1,35 +0,0 @@ -package resolver - -import ( - "context" - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" -) - -// FrostFSResolver represents virtual connection to the FrostFS network. -// It implements resolver.FrostFS. -type FrostFSResolver struct { - pool *pool.Pool -} - -// NewFrostFSResolver creates new FrostFSResolver using provided pool.Pool. -func NewFrostFSResolver(p *pool.Pool) *FrostFSResolver { - return &FrostFSResolver{pool: p} -} - -// SystemDNS implements resolver.FrostFS interface method. -func (x *FrostFSResolver) SystemDNS(ctx context.Context) (string, error) { - networkInfo, err := x.pool.NetworkInfo(ctx) - if err != nil { - return "", fmt.Errorf("read network info via client: %w", err) - } - - domain := networkInfo.RawNetworkParameter("SystemDNS") - if domain == nil { - return "", errors.New("system DNS parameter not found or empty") - } - - return string(domain), nil -} From f20ea67b4663a1a97bbf5bfa122679a8bd1d0247 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 16 Jul 2024 14:58:47 +0300 Subject: [PATCH 461/548] Release v0.30.0 Signed-off-by: Alex Vanin --- CHANGELOG.md | 38 +++++++++++++++++++++++++++++--------- VERSION | 2 +- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 105ac41..cf47b00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,19 +4,30 @@ This document outlines major changes between releases. ## [Unreleased] -## [0.28.1] - 2024-01-24 +## [0.30.0] - Kangshung - 2024-07-22 + +### Fixed +- Handle query unescape and invalid bearer token errors (#107) +- Fix HTTP/2 requests (#110) ### Added -- Tree pool traversal limit (#92) - Add new `reconnect_interval` config param (#100) +- Erasure coding support in placement policy (#114) +- HTTP Header canonicalizer for well-known headers (#121) -### Update from 0.28.0 -See new `frostfs.tree_pool_max_attempts` config parameter. +### Changed +- Improve test coverage (#112, #117) +- Bumped vulnerable dependencies (#115) +- Replace extended ACL examples with policies in README (#118) + +### Removed + +## [0.29.0] - Zemu - 2024-05-27 ### Fixed - Fix possibility of panic during SIGHUP (#99) -- Handle query unescape and invalid bearer token errors (#107) -- Fix HTTP/2 requests (#110) +- Handle query unescape and invalid bearer token errors (#108) +- Fix log-level change on SIGHUP (#105) ### Added - Support client side object cut (#70) @@ -24,12 +35,19 @@ See new `frostfs.tree_pool_max_attempts` config parameter. - Add `frostfs.buffer_max_size_for_put` config param - Add bucket/container caching - Disable homomorphic hash for PUT if it's disabled in container itself -- Add new `logger.destination` config param (#89) +- Add new `logger.destination` config param with journald support (#89, #104) - Add support namespaces (#91) ### Changed +- Replace atomics with mutex for reloadable params (#74) -### Removed +## [0.28.1] - 2024-01-24 + +### Added +- Tree pool traversal limit (#92) + +### Update from 0.28.0 +See new `frostfs.tree_pool_max_attempts` config parameter. ## [0.28.0] - Academy of Sciences - 2023-12-07 @@ -100,4 +118,6 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.27.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/72734ab4...v0.27.0 [0.28.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.27.0...v0.28.0 [0.28.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.0...v0.28.1 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.1...master +[0.29.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.1...v0.29.0 +[0.30.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.29.0...v0.30.0 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.0...master diff --git a/VERSION b/VERSION index 244df55..9388ecb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.28.1 +v0.30.0 From fcf99d9a599a6dd39b7fc8a5c0cd10970c8255eb Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 15 Jul 2024 16:35:08 +0300 Subject: [PATCH 462/548] [#127] Split FrostFS ReadObject to separate methods Signed-off-by: Denis Kirillov --- internal/frostfs/frostfs.go | 56 ++++++++++++++++++-------------- internal/handler/download.go | 6 ++-- internal/handler/frostfs_mock.go | 51 +++++++++++++++++++---------- internal/handler/handler.go | 34 +++++++++++++++---- internal/handler/head.go | 20 +++++------- internal/handler/reader.go | 10 +++--- 6 files changed, 109 insertions(+), 68 deletions(-) diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go index fc41420..2610564 100644 --- a/internal/frostfs/frostfs.go +++ b/internal/frostfs/frostfs.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "google.golang.org/grpc/codes" @@ -76,8 +77,25 @@ func (x payloadReader) Read(p []byte) (int, error) { return n, handleObjectError("read payload", err) } -// ReadObject implements frostfs.FrostFS interface method. -func (x *FrostFS) ReadObject(ctx context.Context, prm handler.PrmObjectRead) (*handler.ObjectPart, error) { +// HeadObject implements frostfs.FrostFS interface method. +func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*object.Object, error) { + var prmHead pool.PrmObjectHead + prmHead.SetAddress(prm.Address) + + if prm.BearerToken != nil { + prmHead.UseBearer(*prm.BearerToken) + } + + res, err := x.pool.HeadObject(ctx, prmHead) + if err != nil { + return nil, handleObjectError("read object header via connection pool", err) + } + + return &res, nil +} + +// GetObject implements frostfs.FrostFS interface method. +func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*handler.Object, error) { var prmGet pool.PrmObjectGet prmGet.SetAddress(prm.Address) @@ -85,27 +103,19 @@ func (x *FrostFS) ReadObject(ctx context.Context, prm handler.PrmObjectRead) (*h prmGet.UseBearer(*prm.BearerToken) } - // The code below must be reworked. It was copied from frostfs-s3-gw - // to create similar mocks for unit and fuzzing tests. - // - // However, this code was changed due to specific of expected responses - // from HTTP gateway. HTTP Gateway requires two types of responses: - // * payload as io.Reader + HEAD request - // * only payload as io.Reader - // Therefore all unused params were deleted and code was simplified. - - if prm.PayloadRange[0]+prm.PayloadRange[1] == 0 { - res, err := x.pool.GetObject(ctx, prmGet) - if err != nil { - return nil, handleObjectError("init full payload range reading via connection pool", err) - } - - return &handler.ObjectPart{ - Payload: res.Payload, - Head: &res.Header, - }, nil + res, err := x.pool.GetObject(ctx, prmGet) + if err != nil { + return nil, handleObjectError("init full object reading via connection pool", err) } + return &handler.Object{ + Header: res.Header, + Payload: res.Payload, + }, nil +} + +// RangeObject implements frostfs.FrostFS interface method. +func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) (io.ReadCloser, error) { var prmRange pool.PrmObjectRange prmRange.SetAddress(prm.Address) prmRange.SetOffset(prm.PayloadRange[0]) @@ -120,9 +130,7 @@ func (x *FrostFS) ReadObject(ctx context.Context, prm handler.PrmObjectRead) (*h return nil, handleObjectError("init payload range reading via connection pool", err) } - return &handler.ObjectPart{ - Payload: payloadReader{&res}, - }, nil + return payloadReader{&res}, nil } // SearchObjects implements frostfs.FrostFS interface method. diff --git a/internal/handler/download.go b/internal/handler/download.go index 480254c..88109a6 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -153,19 +153,19 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { } func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error { - prm := PrmObjectRead{ + prm := PrmObjectGet{ PrmAuth: PrmAuth{ BearerToken: btoken, }, Address: addr, } - resGet, err := h.frostfs.ReadObject(ctx, prm) + resGet, err := h.frostfs.GetObject(ctx, prm) if err != nil { return fmt.Errorf("get FrostFS object: %v", err) } - objWriter, err := h.addObjectToZip(zipWriter, resGet.Head) + objWriter, err := h.addObjectToZip(zipWriter, &resGet.Header) if err != nil { return fmt.Errorf("zip create header: %v", err) } diff --git a/internal/handler/frostfs_mock.go b/internal/handler/frostfs_mock.go index eb59fb6..9f4378a 100644 --- a/internal/handler/frostfs_mock.go +++ b/internal/handler/frostfs_mock.go @@ -78,32 +78,49 @@ func (t *TestFrostFS) requestOwner(btoken *bearer.Token) user.ID { return owner } -func (t *TestFrostFS) ReadObject(_ context.Context, prm PrmObjectRead) (*ObjectPart, error) { - sAddr := prm.Address.EncodeToString() +func (t *TestFrostFS) retrieveObject(addr oid.Address, btoken *bearer.Token) (*object.Object, error) { + sAddr := addr.EncodeToString() if obj, ok := t.objects[sAddr]; ok { - owner := t.requestOwner(prm.BearerToken) + owner := t.requestOwner(btoken) - if !t.isAllowed(prm.Address.Container(), owner, acl.OpObjectGet, prm.Address.Object()) { + if !t.isAllowed(addr.Container(), owner, acl.OpObjectGet, addr.Object()) { return nil, ErrAccessDenied } - payload := obj.Payload() - - if prm.PayloadRange[0]+prm.PayloadRange[1] > 0 { - off := prm.PayloadRange[0] - payload = payload[off : off+prm.PayloadRange[1]] - obj = nil // GetRange does not return header values - } - - return &ObjectPart{ - Head: obj, - Payload: io.NopCloser(bytes.NewReader(payload)), - }, nil + return obj, nil } - return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, prm.Address) + return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, addr) } + +func (t *TestFrostFS) HeadObject(_ context.Context, prm PrmObjectHead) (*object.Object, error) { + return t.retrieveObject(prm.Address, prm.BearerToken) +} + +func (t *TestFrostFS) GetObject(_ context.Context, prm PrmObjectGet) (*Object, error) { + obj, err := t.retrieveObject(prm.Address, prm.BearerToken) + if err != nil { + return nil, err + } + + return &Object{ + Header: *obj, + Payload: io.NopCloser(bytes.NewReader(obj.Payload())), + }, nil +} + +func (t *TestFrostFS) RangeObject(_ context.Context, prm PrmObjectRange) (io.ReadCloser, error) { + obj, err := t.retrieveObject(prm.Address, prm.BearerToken) + if err != nil { + return nil, err + } + + off := prm.PayloadRange[0] + payload := obj.Payload()[off : off+prm.PayloadRange[1]] + return io.NopCloser(bytes.NewReader(payload)), nil +} + func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID, error) { b := make([]byte, 32) if _, err := io.ReadFull(rand.Reader, b); err != nil { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 197649e..0bbcdb9 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -46,8 +46,26 @@ type PrmAuth struct { BearerToken *bearer.Token } -// PrmObjectRead groups parameters of FrostFS.ReadObject operation. -type PrmObjectRead struct { +// PrmObjectHead groups parameters of FrostFS.HeadObject operation. +type PrmObjectHead struct { + // Authentication parameters. + PrmAuth + + // Address to read the object header from. + Address oid.Address +} + +// PrmObjectGet groups parameters of FrostFS.GetObject operation. +type PrmObjectGet struct { + // Authentication parameters. + PrmAuth + + // Address to read the object header from. + Address oid.Address +} + +// PrmObjectRange groups parameters of FrostFS.RangeObject operation. +type PrmObjectRange struct { // Authentication parameters. PrmAuth @@ -58,10 +76,10 @@ type PrmObjectRead struct { PayloadRange [2]uint64 } -// ObjectPart represents partially read FrostFS object. -type ObjectPart struct { - // Object header with optional in-memory payload part. - Head *object.Object +// Object represents FrostFS object. +type Object struct { + // Object header (doesn't contain payload). + Header object.Object // Object payload part encapsulated in io.Reader primitive. // Returns ErrAccessDenied on read access violation. @@ -115,7 +133,9 @@ var ( // FrostFS represents virtual connection to FrostFS network. type FrostFS interface { Container(context.Context, PrmContainer) (*container.Container, error) - ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error) + HeadObject(context.Context, PrmObjectHead) (*object.Object, error) + GetObject(context.Context, PrmObjectGet) (*Object, error) + RangeObject(context.Context, PrmObjectRange) (io.ReadCloser, error) CreateObject(context.Context, PrmObjectCreate) (oid.ID, error) SearchObjects(context.Context, PrmObjectSearch) (ResObjectSearch, error) utils.EpochInfoFetcher diff --git a/internal/handler/head.go b/internal/handler/head.go index 96d1f49..f0a1e94 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -29,22 +29,22 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid btoken := bearerToken(ctx) - prm := PrmObjectRead{ + prm := PrmObjectHead{ PrmAuth: PrmAuth{ BearerToken: btoken, }, Address: objectAddress, } - obj, err := h.frostfs.ReadObject(ctx, prm) + obj, err := h.frostfs.HeadObject(ctx, prm) if err != nil { req.handleFrostFSErr(err, start) return } - req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.Head.PayloadSize(), 10)) + req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10)) var contentType string - for _, attr := range obj.Head.Attributes() { + for _, attr := range obj.Attributes() { key := attr.Key() val := attr.Value() if !isValidToken(key) || !isValidValue(val) { @@ -70,11 +70,11 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid } } - idsToResponse(&req.Response, obj.Head) + idsToResponse(&req.Response, obj) if len(contentType) == 0 { - contentType, _, err = readContentType(obj.Head.PayloadSize(), func(sz uint64) (io.Reader, error) { - prmRange := PrmObjectRead{ + contentType, _, err = readContentType(obj.PayloadSize(), func(sz uint64) (io.Reader, error) { + prmRange := PrmObjectRange{ PrmAuth: PrmAuth{ BearerToken: btoken, }, @@ -82,11 +82,7 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid PayloadRange: [2]uint64{0, sz}, } - resObj, err := h.frostfs.ReadObject(ctx, prmRange) - if err != nil { - return nil, err - } - return resObj.Payload, nil + return h.frostfs.RangeObject(ctx, prmRange) }) if err != nil && err != io.EOF { req.handleFrostFSErr(err, start) diff --git a/internal/handler/reader.go b/internal/handler/reader.go index 81694bc..65d258b 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -55,14 +55,14 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi filename string ) - prm := PrmObjectRead{ + prm := PrmObjectGet{ PrmAuth: PrmAuth{ BearerToken: bearerToken(ctx), }, Address: objectAddress, } - rObj, err := h.frostfs.ReadObject(ctx, prm) + rObj, err := h.frostfs.GetObject(ctx, prm) if err != nil { req.handleFrostFSErr(err, start) return @@ -74,11 +74,11 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi dis = "attachment" } - payloadSize := rObj.Head.PayloadSize() + payloadSize := rObj.Header.PayloadSize() req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) var contentType string - for _, attr := range rObj.Head.Attributes() { + for _, attr := range rObj.Header.Attributes() { key := attr.Key() val := attr.Value() if !isValidToken(key) || !isValidValue(val) { @@ -107,7 +107,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi } } - idsToResponse(&req.Response, rObj.Head) + idsToResponse(&req.Response, &rObj.Header) if len(contentType) == 0 { // determine the Content-Type from the payload head From 5ee09790f05cc70cd0bf274d996435ba03a56554 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Fri, 16 Aug 2024 12:56:38 +0300 Subject: [PATCH 463/548] [#126] Fix docker warnings Signed-off-by: Roman Loginov --- .docker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index d39fba1..f45c864 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -1,9 +1,9 @@ -FROM golang:1.22-alpine as basebuilder +FROM golang:1.22-alpine AS basebuilder RUN apk add --update make bash ca-certificates -FROM basebuilder as builder -ENV GOGC off -ENV CGO_ENABLED 0 +FROM basebuilder AS builder +ENV GOGC=off +ENV CGO_ENABLED=0 ARG BUILD=now ARG VERSION=dev ARG REPO=repository From 151e5bc1c835a62f1d950647031109b4f02530f8 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 23 Aug 2024 13:19:08 +0300 Subject: [PATCH 464/548] [#132] Update Go version Signed-off-by: Nikita Zinkevich --- .forgejo/workflows/builds.yml | 2 +- .forgejo/workflows/dco.yml | 2 +- .forgejo/workflows/tests.yml | 6 +++--- .golangci.yml | 3 ++- .pre-commit-config.yaml | 5 ----- CHANGELOG.md | 3 +++ Makefile | 4 ++-- go.mod | 2 +- internal/handler/reader_test.go | 2 +- utils/tracing.go | 4 ++-- 10 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.forgejo/workflows/builds.yml b/.forgejo/workflows/builds.yml index 17f1f2e..490a97c 100644 --- a/.forgejo/workflows/builds.yml +++ b/.forgejo/workflows/builds.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.21', '1.22' ] + go_versions: [ '1.22', '1.23' ] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index eb23ec5..4acd633 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.23' - name: Run commit format checker uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 74e0b2c..db7f986 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.23' cache: true - name: Install linters @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.21', '1.22' ] + go_versions: [ '1.22', '1.23' ] fail-fast: false steps: - uses: actions/checkout@v3 @@ -38,4 +38,4 @@ jobs: run: make dep - name: Run tests - run: make test \ No newline at end of file + run: make test diff --git a/.golangci.yml b/.golangci.yml index 5459bde..d9f93eb 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -12,7 +12,8 @@ run: # output configuration options output: # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" - format: tab + formats: + - format: tab # all available settings of specific linters linters-settings: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e97fc23..3c963be 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,11 +30,6 @@ repos: hooks: - id: shellcheck - - repo: https://github.com/golangci/golangci-lint - rev: v1.51.2 - hooks: - - id: golangci-lint - - repo: local hooks: - id: make-lint-install diff --git a/CHANGELOG.md b/CHANGELOG.md index cf47b00..b322c96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ This document outlines major changes between releases. ## [Unreleased] +### Changed +- Update go version to 1.22 (#132) + ## [0.30.0] - Kangshung - 2024-07-22 ### Fixed diff --git a/Makefile b/Makefile index 04cfea4..372b89b 100755 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") GO_VERSION ?= 1.22 -LINT_VERSION ?= 1.54.0 -TRUECLOUDLAB_LINT_VERSION ?= 0.0.2 +LINT_VERSION ?= 1.60.3 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.6 BUILD ?= $(shell date -u --iso=seconds) HUB_IMAGE ?= truecloudlab/frostfs-http-gw diff --git a/go.mod b/go.mod index 493a003..4e4bf9c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw -go 1.21 +go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 diff --git a/internal/handler/reader_test.go b/internal/handler/reader_test.go index 73899ca..c63a734 100644 --- a/internal/handler/reader_test.go +++ b/internal/handler/reader_test.go @@ -35,7 +35,7 @@ func TestDetector(t *testing.T) { } { t.Run(tc.Name, func(t *testing.T) { contentType, data, err := readContentType(uint64(len(tc.Expected)), - func(sz uint64) (io.Reader, error) { + func(uint64) (io.Reader, error) { return strings.NewReader(tc.Expected), nil }, ) diff --git a/utils/tracing.go b/utils/tracing.go index 14c059a..c8e467d 100644 --- a/utils/tracing.go +++ b/utils/tracing.go @@ -30,12 +30,12 @@ func (c *httpCarrier) Set(key string, value string) { func (c *httpCarrier) Keys() []string { dict := make(map[string]interface{}) c.r.Request.Header.VisitAll( - func(key, value []byte) { + func(key, _ []byte) { dict[string(key)] = true }, ) c.r.Response.Header.VisitAll( - func(key, value []byte) { + func(key, _ []byte) { dict[string(key)] = true }, ) From ca426fff4df117b5ce119d7a5f123b11cc99a7a1 Mon Sep 17 00:00:00 2001 From: Roman Ognev Date: Fri, 30 Aug 2024 20:06:01 +0300 Subject: [PATCH 465/548] [#135] Add fuzzing tests for handlers Signed-off-by: Roman Ognev --- Makefile | 36 +- README.md | 23 + go.mod | 1 + go.sum | 2 + internal/handler/handler_fuzz_test.go | 580 ++++++++++++++++++++++++++ 5 files changed, 641 insertions(+), 1 deletion(-) create mode 100644 internal/handler/handler_fuzz_test.go diff --git a/Makefile b/Makefile index 372b89b..c1f4f50 100755 --- a/Makefile +++ b/Makefile @@ -30,6 +30,11 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed "s/-/~/")-${OS_RELEASE} .PHONY: debpackage debclean +FUZZ_NGFUZZ_DIR ?= "" +FUZZ_TIMEOUT ?= 30 +FUZZ_FUNCTIONS ?= "all" +FUZZ_AUX ?= "" + # Make all binaries all: $(BINS) $(BINS): $(DIRS) dep @@ -78,6 +83,35 @@ cover: @go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic @go tool cover -html=coverage.txt -o coverage.html +# Run fuzzing +CLANG := $(shell which clang-17 2>/dev/null) +.PHONY: check-clang all +check-clang: +ifeq ($(CLANG),) + @echo "clang-17 is not installed. Please install it before proceeding - https://apt.llvm.org/llvm.sh " + @exit 1 +endif + +.PHONY: check-ngfuzz all +check-ngfuzz: + @if [ -z "$(FUZZ_NGFUZZ_DIR)" ]; then \ + echo "Please set a variable FUZZ_NGFUZZ_DIR to specify path to the ngfuzz"; \ + exit 1; \ + fi + +.PHONY: install-fuzzing-deps +install-fuzzing-deps: check-clang check-ngfuzz + +.PHONY: fuzz +fuzz: install-fuzzing-deps + @START_PATH=$$(pwd); \ + ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \ + cd $(FUZZ_NGFUZZ_DIR) && \ + ./ngfuzz -clean && \ + ./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \ + ./ngfuzz -report + + # Reformat code fmt: @echo "⇒ Processing gofmt check" @@ -149,7 +183,7 @@ version: # Clean up clean: rm -rf vendor - rm -rf $(BINDIR) + rm -rf $(BINDIR) # Package for Debian debpackage: diff --git a/README.md b/README.md index aa982db..019b8ff 100644 --- a/README.md +++ b/README.md @@ -575,3 +575,26 @@ See [configuration](./docs/gate-configuration.md). ## Credits Please see [CREDITS](CREDITS.md) for details. + +## Fuzzing + +To run fuzzing tests use the following command: + +```shell +$ make fuzz +``` + +This command will install dependencies for the fuzzing process and run existing fuzzing tests. + +You can also use the following arguments: + +``` +FUZZ_TIMEOUT - time to run each fuzzing test (default 30) +FUZZ_FUNCTIONS - fuzzing tests that will be started (default "all") +FUZZ_AUX - additional parameters for the fuzzer (for example, "-debug") +FUZZ_NGFUZZ_DIR - path to ngfuzz tool +```` + +## Credits + +Please see [CREDITS](CREDITS.md) for details. diff --git a/go.mod b/go.mod index 4e4bf9c..accedfb 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.13.0 + github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 github.com/valyala/fasthttp v1.34.0 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 diff --git a/go.sum b/go.sum index 73b2798..07eccfa 100644 --- a/go.sum +++ b/go.sum @@ -820,6 +820,8 @@ github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBN github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 h1:GpfJ7OdNjS7BFTVwNCUI9L4aCJOFRbr5fdHqjdhoYE8= +github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4/go.mod h1:f3jBhpWvuZmue0HZK52GzRHJOYHYSILs/c8+K2S/J+o= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= diff --git a/internal/handler/handler_fuzz_test.go b/internal/handler/handler_fuzz_test.go new file mode 100644 index 0000000..ad2ae6e --- /dev/null +++ b/internal/handler/handler_fuzz_test.go @@ -0,0 +1,580 @@ +//go:build gofuzz +// +build gofuzz + +package handler + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "mime/multipart" + "net/http" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + go_fuzz_utils "github.com/trailofbits/go-fuzz-utils" + "github.com/valyala/fasthttp" +) + +const ( + fuzzSuccessExitCode = 0 + fuzzFailExitCode = -1 +) + +func prepareStrings(tp *go_fuzz_utils.TypeProvider, count int) ([]string, error) { + array := make([]string, count) + var err error + + for i := 0; i < count; i++ { + err = tp.Reset() + if err != nil { + return nil, err + } + + array[i], err = tp.GetString() + if err != nil { + return nil, err + } + } + + return array, nil +} + +func prepareBools(tp *go_fuzz_utils.TypeProvider, count int) ([]bool, error) { + array := make([]bool, count) + var err error + + for i := 0; i < count; i++ { + err = tp.Reset() + if err != nil { + return nil, err + } + + array[i], err = tp.GetBool() + if err != nil { + return nil, err + } + } + + return array, nil +} + +func getRandomDeterministicPositiveIntInRange(tp *go_fuzz_utils.TypeProvider, max int) (int, error) { + count, err := tp.GetInt() + if err != nil { + return -1, err + } + count = count % max + if count < 0 { + count += max + } + return count, nil +} + +func generateHeaders(tp *go_fuzz_utils.TypeProvider, r *fasthttp.Request, params []string) error { + count, err := tp.GetInt() + if err != nil { + return err + } + count = count % len(params) + if count < 0 { + count += len(params) + } + + for i := 0; i < count; i++ { + position, err := tp.GetInt() + if err != nil { + return err + } + position = position % len(params) + if position < 0 { + position += len(params) + } + + v, err := tp.GetString() + if err != nil { + return err + } + + r.Header.Set(params[position], v) + + } + + return nil +} + +func maybeFillRandom(tp *go_fuzz_utils.TypeProvider, initValue string) (string, error) { + rnd, err := tp.GetBool() + if err != nil { + return "", err + } + if rnd == true { + initValue, err = tp.GetString() + if err != nil { + return "", err + } + } + return initValue, nil +} + +func upload(tp *go_fuzz_utils.TypeProvider) (context.Context, *handlerContext, cid.ID, *fasthttp.RequestCtx, string, string, string, error) { + hc, err := prepareHandlerContext() + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + aclList := []acl.Basic{ + acl.Private, + acl.PrivateExtended, + acl.PublicRO, + acl.PublicROExtended, + acl.PublicRW, + acl.PublicRWExtended, + acl.PublicAppend, + acl.PublicAppendExtended, + } + + pos, err := getRandomDeterministicPositiveIntInRange(tp, len(aclList)) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + acl := aclList[pos] + + strings, err := prepareStrings(tp, 6) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + bktName := strings[0] + objFileName := strings[1] + valAttr := strings[2] + keyAttr := strings[3] + + if len(bktName) == 0 { + return nil, nil, cid.ID{}, nil, "", "", "", errors.New("not enought buckets") + } + + cnrID, cnr, err := hc.prepareContainer(bktName, acl) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + hc.frostfs.SetContainer(cnrID, cnr) + + ctx := context.Background() + ctx = middleware.SetNamespace(ctx, "") + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cnrID.EncodeToString()) + + attributes := map[string]string{ + object.AttributeFileName: objFileName, + keyAttr: valAttr, + } + + var buff bytes.Buffer + w := multipart.NewWriter(&buff) + fw, err := w.CreateFormFile("file", attributes[object.AttributeFileName]) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + content, err := tp.GetBytes() + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + if _, err = io.Copy(fw, bytes.NewReader(content)); err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + if err = w.Close(); err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + r.Request.SetBodyStream(&buff, buff.Len()) + r.Request.Header.Set("Content-Type", w.FormDataContentType()) + r.Request.Header.Set("X-Attribute-"+keyAttr, valAttr) + + err = generateHeaders(tp, &r.Request, []string{"X-Attribute-", "X-Attribute-DupKey", "X-Attribute-MyAttribute", "X-Attribute-System-DupKey", "X-Attribute-System-Expiration-Epoch1", "X-Attribute-SYSTEM-Expiration-Epoch2", "X-Attribute-system-Expiration-Epoch3", "X-Attribute-User-Attribute", "X-Attribute-", "X-Attribute-FileName", "X-Attribute-FROSTFS", "X-Attribute-neofs", "X-Attribute-SYSTEM", "X-Attribute-System-Expiration-Duration", "X-Attribute-System-Expiration-Epoch", "X-Attribute-System-Expiration-RFC3339", "X-Attribute-System-Expiration-Timestamp", "X-Attribute-Timestamp", "X-Attribute-" + strings[4], "X-Attribute-System-" + strings[5]}) + if err != nil { + return nil, nil, cid.ID{}, nil, "", "", "", err + } + + hc.Handler().Upload(r) + + if r.Response.StatusCode() != http.StatusOK { + return nil, nil, cid.ID{}, nil, "", "", "", errors.New("error on upload") + } + + return ctx, hc, cnrID, r, objFileName, keyAttr, valAttr, nil +} + +func InitFuzzUpload() { + +} + +func DoFuzzUpload(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + _, _, _, _, _, _, _, err = upload(tp) + if err != nil { + return fuzzFailExitCode + } + + return fuzzSuccessExitCode +} + +func FuzzUpload(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzUpload(data) + }) +} + +func downloadOrHead(tp *go_fuzz_utils.TypeProvider, ctx context.Context, hc *handlerContext, cnrID cid.ID, resp *fasthttp.RequestCtx, filename string) (*fasthttp.RequestCtx, error) { + + var putRes putResponse + + defer func() { + if r := recover(); r != nil { + panic(resp) + } + }() + + data := resp.Response.Body() + err := json.Unmarshal(data, &putRes) + + if err != nil { + return nil, err + } + + obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] + attr := object.NewAttribute() + attr.SetKey(object.AttributeFilePath) + + filename, err = maybeFillRandom(tp, filename) + if err != nil { + return nil, err + } + + attr.SetValue(filename) + obj.SetAttributes(append(obj.Attributes(), *attr)...) + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return nil, err + } + oid := putRes.ObjectID + oid, err = maybeFillRandom(tp, oid) + if err != nil { + return nil, err + } + r.SetUserValue("cid", cid) + r.SetUserValue("oid", oid) + + rnd, err := tp.GetBool() + if err != nil { + return nil, err + } + if rnd == true { + r.SetUserValue("download", "true") + } + + return r, nil +} + +func InitFuzzGet() { + +} + +func DoFuzzGet(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, resp, filename, _, _, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename) + if err != nil { + return fuzzFailExitCode + } + + hc.Handler().DownloadByAddressOrBucketName(r) + + return fuzzSuccessExitCode +} + +func FuzzGet(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzUpload(data) + }) +} + +func InitFuzzHead() { + +} + +func DoFuzzHead(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, resp, filename, _, _, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + r, err := downloadOrHead(tp, ctx, hc, cnrID, resp, filename) + if err != nil { + return fuzzFailExitCode + } + + hc.Handler().HeadByAddressOrBucketName(r) + + return fuzzSuccessExitCode +} + +func FuzzHead(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzHead(data) + }) +} + +func InitFuzzDownloadByAttribute() { + +} + +func DoFuzzDownloadByAttribute(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return fuzzFailExitCode + } + + attrKey, err = maybeFillRandom(tp, attrKey) + if err != nil { + return fuzzFailExitCode + } + + attrVal, err = maybeFillRandom(tp, attrVal) + if err != nil { + return fuzzFailExitCode + } + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cid) + r.SetUserValue("attr_key", attrKey) + r.SetUserValue("attr_val", attrVal) + + hc.Handler().DownloadByAttribute(r) + + return fuzzSuccessExitCode +} + +func FuzzDownloadByAttribute(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzDownloadByAttribute(data) + }) +} + +func InitFuzzHeadByAttribute() { + +} + +func DoFuzzHeadByAttribute(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, _, _, attrKey, attrVal, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return fuzzFailExitCode + } + + attrKey, err = maybeFillRandom(tp, attrKey) + if err != nil { + return fuzzFailExitCode + } + + attrVal, err = maybeFillRandom(tp, attrVal) + if err != nil { + return fuzzFailExitCode + } + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cid) + r.SetUserValue("attr_key", attrKey) + r.SetUserValue("attr_val", attrVal) + + hc.Handler().HeadByAttribute(r) + + return fuzzSuccessExitCode +} + +func FuzzHeadByAttribute(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzHeadByAttribute(data) + }) +} + +func InitFuzzDownloadZipped() { + +} + +func DoFuzzDownloadZipped(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + ctx, hc, cnrID, _, _, _, _, err := upload(tp) + if err != nil { + return fuzzFailExitCode + } + + cid := cnrID.EncodeToString() + cid, err = maybeFillRandom(tp, cid) + if err != nil { + return fuzzFailExitCode + } + + prefix := "" + prefix, err = maybeFillRandom(tp, prefix) + if err != nil { + return fuzzFailExitCode + } + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + r.SetUserValue("cid", cid) + r.SetUserValue("prefix", prefix) + + hc.Handler().DownloadZipped(r) + + return fuzzSuccessExitCode +} + +func FuzzDownloadZipped(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzDownloadZipped(data) + }) +} + +func InitFuzzStoreBearerTokenAppCtx() { + +} + +func DoFuzzStoreBearerTokenAppCtx(input []byte) int { + // FUZZER INIT + if len(input) < 100 { + return fuzzFailExitCode + } + + tp, err := go_fuzz_utils.NewTypeProvider(input) + if err != nil { + return fuzzFailExitCode + } + + prefix := "" + prefix, err = maybeFillRandom(tp, prefix) + if err != nil { + return fuzzFailExitCode + } + + ctx := context.Background() + ctx = middleware.SetNamespace(ctx, "") + + r := new(fasthttp.RequestCtx) + utils.SetContextToRequest(ctx, r) + + strings, err := prepareStrings(tp, 3) + + rand, err := prepareBools(tp, 2) + + if rand[0] == true { + r.Request.Header.Set(fasthttp.HeaderAuthorization, "Bearer"+strings[0]) + } else if rand[1] == true { + r.Request.Header.SetCookie(fasthttp.HeaderAuthorization, "Bearer"+strings[1]) + } else { + r.Request.Header.Set(fasthttp.HeaderAuthorization, "Bearer"+strings[0]) + r.Request.Header.SetCookie(fasthttp.HeaderAuthorization, "Bearer"+strings[1]) + } + + tokens.StoreBearerTokenAppCtx(ctx, r) + + return fuzzSuccessExitCode +} + +func FuzzStoreBearerTokenAppCtx(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + DoFuzzStoreBearerTokenAppCtx(data) + }) +} From 77ffde58e94ccc0ca9a7f20485c142b4d2769ff7 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Fri, 30 Aug 2024 13:02:49 +0300 Subject: [PATCH 466/548] [#123] Add SECURITY.md Signed-off-by: Pavel Pogodaev --- SECURITY.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..46fe535 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,26 @@ +# Security Policy + + +## How To Report a Vulnerability + +If you think you have found a vulnerability in this repository, please report it to us through coordinated disclosure. + +**Please do not report security vulnerabilities through public issues, discussions, or change requests.** + +Instead, you can report it using one of the following ways: + +* Contact the [TrueCloudLab Security Team](mailto:security@frostfs.info) via email + +Please include as much of the information listed below as you can to help us better understand and resolve the issue: + +* The type of issue (e.g., buffer overflow, or cross-site scripting) +* Affected version(s) +* Impact of the issue, including how an attacker might exploit the issue +* Step-by-step instructions to reproduce the issue +* The location of the affected source code (tag/branch/commit or direct URL) +* Full paths of source file(s) related to the manifestation of the issue +* Any special configuration required to reproduce the issue +* Any log files that are related to this issue (if possible) +* Proof-of-concept or exploit code (if possible) + +This information will help us triage your report more quickly. From 843708a558c5b9f308c8eafeaa09171a45500202 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Mon, 26 Aug 2024 18:47:33 +0300 Subject: [PATCH 467/548] [#134] Support percent-encoding Signed-off-by: Pavel Pogodaev --- CHANGELOG.md | 1 + internal/handler/handler.go | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b322c96..a489027 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ This document outlines major changes between releases. ## [Unreleased] +- Support percent-encoding for GET queries (#134) ### Changed - Update go version to 1.22 (#132) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 0bbcdb9..4de9d9a 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -215,6 +215,12 @@ func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, log = h.log.With(zap.String("bucketname", bucketname), zap.String("key", key)) ) + unescapedKey, err := url.QueryUnescape(key) + if err != nil { + logAndSendBucketError(req, log, err) + return + } + ctx := utils.GetContextFromRequest(req) bktInfo, err := h.getBucketInfo(ctx, bucketname, log) @@ -223,7 +229,7 @@ func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, return } - foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, key) + foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey) if err != nil { if errors.Is(err, tree.ErrNodeAccessDenied) { response.Error(req, "Access Denied", fasthttp.StatusForbidden) From 7e80f0cce6fba901414964347e0ccf7755ff8553 Mon Sep 17 00:00:00 2001 From: Aleksey Savaitan Date: Tue, 10 Sep 2024 10:09:51 +0300 Subject: [PATCH 468/548] [#139] Add root ca cert for telemetry configuration Signed-off-by: Aleksey Savaitan --- cmd/http-gw/app.go | 17 ++++++ cmd/http-gw/settings.go | 7 ++- config/config.env | 1 + config/config.yaml | 1 + docs/gate-configuration.md | 12 ++-- go.mod | 47 +++++++-------- go.sum | 119 ++++++++++++++++--------------------- 7 files changed, 102 insertions(+), 102 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 561598f..4c49ee4 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "crypto/x509" "errors" "fmt" "net/http" @@ -737,6 +738,22 @@ func (a *app) initTracing(ctx context.Context) { InstanceID: instanceID, Version: Version, } + + if trustedCa := a.cfg.GetString(cfgTracingTrustedCa); trustedCa != "" { + caBytes, err := os.ReadFile(trustedCa) + if err != nil { + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + return + } + certPool := x509.NewCertPool() + ok := certPool.AppendCertsFromPEM(caBytes) + if !ok { + a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert")) + return + } + cfg.ServerCaCertPool = certPool + } + updated, err := tracing.Setup(ctx, cfg) if err != nil { a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 0d97dcb..3fe0023 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -75,9 +75,10 @@ const ( cfgPprofAddress = "pprof.address" // Tracing ... - cfgTracingEnabled = "tracing.enabled" - cfgTracingExporter = "tracing.exporter" - cfgTracingEndpoint = "tracing.endpoint" + cfgTracingEnabled = "tracing.enabled" + cfgTracingExporter = "tracing.exporter" + cfgTracingEndpoint = "tracing.endpoint" + cfgTracingTrustedCa = "tracing.trusted_ca" // Pool config. cfgConTimeout = "connect_timeout" diff --git a/config/config.env b/config/config.env index 05b83b3..b7347d7 100644 --- a/config/config.env +++ b/config/config.env @@ -99,6 +99,7 @@ HTTP_GW_ZIP_COMPRESSION=false HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENDPOINT="localhost:4317" HTTP_GW_TRACING_EXPORTER="otlp_grpc" +HTTP_GW_TRACING_TRUSTED_CA="" HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 diff --git a/config/config.yaml b/config/config.yaml index 7f8077b..ef57612 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -13,6 +13,7 @@ tracing: enabled: true exporter: "otlp_grpc" endpoint: "localhost:4317" + trusted_ca: "" logger: level: debug # Log level. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 8e3daad..e93e6cf 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -256,13 +256,15 @@ tracing: enabled: true exporter: "otlp_grpc" endpoint: "localhost:4317" + trusted_ca: "/etc/ssl/telemetry-trusted-ca.pem" ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|------------|----------|---------------|------------------|---------------------------------------------------------------| -| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | -| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | -| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|--------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| +| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | +| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | +| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | +| `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | # `runtime` section Contains runtime parameters. diff --git a/go.mod b/go.mod index accedfb..3519a14 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 @@ -19,12 +19,12 @@ require ( github.com/testcontainers/testcontainers-go v0.13.0 github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 github.com/valyala/fasthttp v1.34.0 - go.opentelemetry.io/otel v1.16.0 - go.opentelemetry.io/otel/trace v1.16.0 + go.opentelemetry.io/otel v1.28.0 + go.opentelemetry.io/otel/trace v1.28.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 - golang.org/x/net v0.23.0 - google.golang.org/grpc v1.62.0 + golang.org/x/net v0.26.0 + google.golang.org/grpc v1.64.0 ) require ( @@ -39,7 +39,7 @@ require ( github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/containerd v1.6.2 // indirect @@ -51,16 +51,15 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/klauspost/compress v1.16.4 // indirect @@ -95,24 +94,22 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect go.etcd.io/bbolt v1.3.9 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 // indirect - go.opentelemetry.io/otel/metric v1.16.0 // indirect - go.opentelemetry.io/otel/sdk v1.16.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 07eccfa..bac691a 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 h1:MV/vKJWLQT34RRbXYvkNKFYGNjL5bRNuCQMXkbC7fLI= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36/go.mod h1:vluJ/+yQMcq8ZIZZSA7Te+JKClr0lgtRErjICvb8wto= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= @@ -137,8 +137,8 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0Bsq github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -160,11 +160,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= @@ -336,7 +332,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fasthttp/router v1.4.1 h1:3xPUO+hy/HAkgGDSd5sX5w18cyGDIFbC7vip8KwPDk8= @@ -366,8 +361,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -402,9 +397,6 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= -github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -436,8 +428,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -504,9 +494,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAszqCJMbfTISJ7oMftp8+UGV08= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= @@ -739,8 +728,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -874,25 +863,23 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= -go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 h1:+XWJd3jf75RXJq29mxbuXhCXFDG3S3R4vBUeSI2P7tE= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0/go.mod h1:hqgzBPTf4yONMFgdZvL/bK42R/iinTyVQtiWihs3SZc= -go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= -go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= -go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= -go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -921,8 +908,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -959,8 +946,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1012,8 +999,8 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1023,7 +1010,6 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1035,8 +1021,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1127,23 +1113,22 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1213,8 +1198,8 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1290,13 +1275,10 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 h1:x9PwdEgd11LgK+orcck69WVRo7DezSO4VUMPI4xpc8A= -google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go.mod h1:rbHMSEDyoYX62nRVLOCc4Qt1HbsdytAYoVwgjiOhF3I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c h1:NUsgEN92SQQqzfA+YtqYNqYmB3DMMYLlIwUZAQFVFbo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1320,9 +1302,8 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1336,8 +1317,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From a4233b006c6ce227078c05938c1f5a235933b7ab Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Mon, 23 Sep 2024 09:09:32 +0300 Subject: [PATCH 469/548] [#144] Update frostfs-sdk-go Signed-off-by: Nikita Zinkevich --- go.mod | 11 +++++++---- go.sum | 22 ++++++++++++++-------- internal/frostfs/frostfs.go | 5 ++++- metrics/metrics.go | 2 -- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 3519a14..afa704b 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.22 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 github.com/fasthttp/router v1.4.1 @@ -24,7 +24,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/net v0.26.0 - google.golang.org/grpc v1.64.0 + google.golang.org/grpc v1.66.2 ) require ( @@ -36,11 +36,12 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/hcsshim v0.9.2 // indirect + github.com/VictoriaMetrics/easyproto v0.1.4 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/containerd v1.6.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -62,8 +63,10 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.16.4 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/sys/mount v0.3.2 // indirect github.com/moby/sys/mountinfo v0.6.1 // indirect diff --git a/go.sum b/go.sum index bac691a..bc433eb 100644 --- a/go.sum +++ b/go.sum @@ -37,16 +37,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 h1:XxvwQKJT/f16qS3df5PBQPRYKkhy0/A7zH6644QpKD0= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e h1:740ABnOBYx4o6jxULHdSSnVW2fYIO35ohg+Uz59sxd0= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 h1:MV/vKJWLQT34RRbXYvkNKFYGNjL5bRNuCQMXkbC7fLI= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36/go.mod h1:vluJ/+yQMcq8ZIZZSA7Te+JKClr0lgtRErjICvb8wto= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 h1:ijUci3thz0EwWkuRJDocW5D1RkVAJlt9xNG4CYepC90= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= @@ -101,6 +101,8 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/VictoriaMetrics/easyproto v0.1.4 h1:r8cNvo8o6sR4QShBXQd1bKw/VVLSQma/V2KhTBPf+Sc= +github.com/VictoriaMetrics/easyproto v0.1.4/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -142,8 +144,8 @@ github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= @@ -523,6 +525,8 @@ github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -564,6 +568,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -1302,8 +1308,8 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go index 2610564..6cf162a 100644 --- a/internal/frostfs/frostfs.go +++ b/internal/frostfs/frostfs.go @@ -60,7 +60,10 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) } idObj, err := x.pool.PutObject(ctx, prmPut) - return idObj, handleObjectError("save object via connection pool", err) + if err != nil { + return oid.ID{}, handleObjectError("save object via connection pool", err) + } + return idObj.ObjectID, nil } // wraps io.ReadCloser and transforms Read errors related to access violation diff --git a/metrics/metrics.go b/metrics/metrics.go index bfb66ee..b516477 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -191,8 +191,6 @@ func (m *poolMetricsCollector) updateRequestsDuration(node pool.NodeStatistic) { m.requestDuration.WithLabelValues(node.Address(), methodGetContainer).Set(float64(node.AverageGetContainer().Milliseconds())) m.requestDuration.WithLabelValues(node.Address(), methodListContainer).Set(float64(node.AverageListContainer().Milliseconds())) m.requestDuration.WithLabelValues(node.Address(), methodDeleteContainer).Set(float64(node.AverageDeleteContainer().Milliseconds())) - m.requestDuration.WithLabelValues(node.Address(), methodGetContainerEacl).Set(float64(node.AverageGetContainerEACL().Milliseconds())) - m.requestDuration.WithLabelValues(node.Address(), methodSetContainerEacl).Set(float64(node.AverageSetContainerEACL().Milliseconds())) m.requestDuration.WithLabelValues(node.Address(), methodEndpointInfo).Set(float64(node.AverageEndpointInfo().Milliseconds())) m.requestDuration.WithLabelValues(node.Address(), methodNetworkInfo).Set(float64(node.AverageNetworkInfo().Milliseconds())) m.requestDuration.WithLabelValues(node.Address(), methodPutObject).Set(float64(node.AveragePutObject().Milliseconds())) From c8473498aeebf65e855badd995118893cbf5f6ba Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Wed, 25 Sep 2024 14:02:32 +0300 Subject: [PATCH 470/548] [#146] Fix of sighup traicing docs Signed-off-by: Roman Loginov --- docs/gate-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index e93e6cf..e0bb336 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -261,7 +261,7 @@ tracing: | Parameter | Type | SIGHUP reload | Default value | Description | |--------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| -| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | +| `enabled` | `bool` | no | `false` | Flag to enable the tracing. | | `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | | `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | | `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | From 77eb4745814640ff34a6b787712a427c1ff93519 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Thu, 26 Sep 2024 17:48:19 +0300 Subject: [PATCH 471/548] [#147] Add sampling configuration Signed-off-by: Pavel Pogodaev --- cmd/http-gw/settings.go | 73 +++++++++++++++++++++++++++----------- config/config.env | 6 +++- config/config.yaml | 5 +++ docs/gate-configuration.md | 17 ++++++--- 4 files changed, 75 insertions(+), 26 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 3fe0023..d9bbc53 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -41,6 +41,8 @@ const ( defaultConnectTimeout = 10 * time.Second defaultStreamTimeout = 10 * time.Second + defaultLoggerSamplerInterval = 1 * time.Second + defaultShutdownTimeout = 15 * time.Second defaultPoolErrorThreshold uint32 = 100 @@ -91,6 +93,11 @@ const ( cfgLoggerLevel = "logger.level" cfgLoggerDestination = "logger.destination" + cfgLoggerSamplingEnabled = "logger.sampling.enabled" + cfgLoggerSamplingInitial = "logger.sampling.initial" + cfgLoggerSamplingThereafter = "logger.sampling.thereafter" + cfgLoggerSamplingInterval = "logger.sampling.interval" + // Wallet. cfgWalletPassphrase = "wallet.passphrase" cfgWalletPath = "wallet.path" @@ -188,6 +195,10 @@ func settings() *viper.Viper { // logger: v.SetDefault(cfgLoggerLevel, "debug") v.SetDefault(cfgLoggerDestination, "stdout") + v.SetDefault(cfgLoggerSamplingEnabled, false) + v.SetDefault(cfgLoggerSamplingThereafter, 100) + v.SetDefault(cfgLoggerSamplingInitial, 100) + v.SetDefault(cfgLoggerSamplingInterval, defaultLoggerSamplerInterval) // pool: v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) @@ -386,9 +397,9 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { switch dest { case destinationStdout: - return newStdoutLogger(lvl) + return newStdoutLogger(v, lvl) case destinationJournald: - return newJournaldLogger(lvl) + return newJournaldLogger(v, lvl) default: panic(fmt.Sprintf("wrong destination for logger: %s", dest)) } @@ -405,39 +416,59 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { // Logger records a stack trace for all messages at or above fatal level. // // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. -func newStdoutLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { - c := zap.NewProductionConfig() - c.Level = zap.NewAtomicLevelAt(lvl) - c.Encoding = "console" - c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder +func newStdoutLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { + stdout := zapcore.AddSync(os.Stderr) + level := zap.NewAtomicLevelAt(lvl) - l, err := c.Build( - zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), - ) - if err != nil { - panic(fmt.Sprintf("build zap logger instance: %v", err)) - } + consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level) + consoleOutCore = samplingEnabling(v, consoleOutCore) - return l, c.Level + l := zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) + return l, level } -func newJournaldLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { - c := zap.NewProductionConfig() - c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - c.Level = zap.NewAtomicLevelAt(lvl) +func newJournaldLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { + level := zap.NewAtomicLevelAt(lvl) - encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields) + encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields) - core := zapjournald.NewCore(c.Level, encoder, &journald.Journal{}, zapjournald.SyslogFields) + core := zapjournald.NewCore(level, encoder, &journald.Journal{}, zapjournald.SyslogFields) coreWithContext := core.With([]zapcore.Field{ zapjournald.SyslogFacility(zapjournald.LogDaemon), zapjournald.SyslogIdentifier(), zapjournald.SyslogPid(), }) + coreWithContext = samplingEnabling(v, coreWithContext) + l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) - return l, c.Level + return l, level +} + +func newLogEncoder() zapcore.Encoder { + c := zap.NewProductionEncoderConfig() + c.EncodeTime = zapcore.ISO8601TimeEncoder + + return zapcore.NewConsoleEncoder(c) +} + +func samplingEnabling(v *viper.Viper, core zapcore.Core) zapcore.Core { + // Zap samples by logging the first cgfLoggerSamplingInitial entries with a given level + // and message within the specified time interval. + // In the above config, only the first cgfLoggerSamplingInitial log entries with the same level and message + // are recorded in cfgLoggerSamplingInterval interval. Every other log entry will be dropped within the interval since + // cfgLoggerSamplingThereafter is specified here. + if v.GetBool(cfgLoggerSamplingEnabled) { + core = zapcore.NewSamplerWithOptions( + core, + v.GetDuration(cfgLoggerSamplingInterval), + v.GetInt(cfgLoggerSamplingInitial), + v.GetInt(cfgLoggerSamplingThereafter), + ) + } + + return core } func getLogLevel(v *viper.Viper) (zapcore.Level, error) { diff --git a/config/config.env b/config/config.env index b7347d7..e1d5e7d 100644 --- a/config/config.env +++ b/config/config.env @@ -14,8 +14,12 @@ HTTP_GW_PPROF_ADDRESS=localhost:8083 HTTP_GW_PROMETHEUS_ENABLED=true HTTP_GW_PROMETHEUS_ADDRESS=localhost:8084 -# Log level. +# Logger. HTTP_GW_LOGGER_LEVEL=debug +HTTP_GW_LOGGER_SAMPLING_ENABLED=false +HTTP_GW_LOGGER_SAMPLING_INITIAL=100 +HTTP_GW_LOGGER_SAMPLING_THEREAFTER=100 +HTTP_GW_LOGGER_SAMPLING_INTERVAL=1s HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:443 HTTP_GW_SERVER_0_TLS_ENABLED=false diff --git a/config/config.yaml b/config/config.yaml index ef57612..1b87fe9 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -18,6 +18,11 @@ tracing: logger: level: debug # Log level. destination: stdout + sampling: + enabled: false + initial: 100 + thereafter: 100 + interval: 1s server: - address: 0.0.0.0:8080 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index e0bb336..fb5ad2f 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -164,12 +164,21 @@ server: logger: level: debug destination: stdout + sampling: + enabled: false + initial: 100 + thereafter: 100 + interval: 1s ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|---------------|----------|---------------|---------------|----------------------------------------------------------------------------------------------------| -| `level` | `string` | yes | `debug` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | -| `destination` | `string` | no | `stdout` | Destination for logger: `stdout` or `journald` | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------------------|------------|---------------|---------------|----------------------------------------------------------------------------------------------------| +| `level` | `string` | yes | `debug` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | +| `destination` | `string` | no | `stdout` | Destination for logger: `stdout` or `journald` | +| `sampling.enabled` | `bool` | no | false | Sampling enabling flag. | +| `sampling.initial` | `int` | no | '100' | Sampling count of first log entries. | +| `sampling.thereafter` | `int` | no | '100' | Sampling count of entries after an `interval`. | +| `sampling.interval` | `duration` | no | '1s' | Sampling interval of messaging similar entries. | # `web` section From 8fe8f2dcc279548c6a8299fac5e3a3097ab9b20d Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Thu, 26 Sep 2024 17:32:27 +0300 Subject: [PATCH 472/548] [#137] Add index page support Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 54 ++++++ cmd/http-gw/settings.go | 6 + config/config.yaml | 7 +- docs/api.md | 13 +- docs/gate-configuration.md | 38 ++-- go.mod | 2 +- internal/api/tree.go | 1 + internal/frostfs/services/pool_wrapper.go | 84 ++++++++- internal/handler/browse.go | 157 +++++++++++++++++ internal/handler/handler.go | 59 +++++-- internal/handler/handler_test.go | 16 ++ internal/handler/reader.go | 7 + internal/handler/upload.go | 34 ++-- internal/handler/utils.go | 21 +++ internal/logs/logs.go | 2 + internal/templates/index.gotmpl | 90 ++++++++++ internal/templates/template.go | 6 + tokens/bearer-token.go | 4 +- tree/tree.go | 204 +++++++++++++++++++--- tree/tree_test.go | 13 +- 20 files changed, 738 insertions(+), 80 deletions(-) create mode 100644 internal/handler/browse.go create mode 100644 internal/templates/index.gotmpl create mode 100644 internal/templates/template.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 4c49ee4..8d5930d 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -6,6 +6,7 @@ import ( "crypto/x509" "errors" "fmt" + "io" "net/http" "os" "os/signal" @@ -23,6 +24,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" @@ -91,6 +93,8 @@ type ( defaultTimestamp bool zipCompression bool clientCut bool + returnIndexPage bool + indexPageTemplate string bufferMaxSizeForPut uint64 namespaceHeader string defaultNamespaces []string @@ -155,6 +159,7 @@ func newApp(ctx context.Context, opt ...Option) App { a.initResolver() a.initMetrics() a.initTracing(ctx) + a.loadIndexPageTemplate() return a } @@ -177,12 +182,59 @@ func (s *appSettings) ZipCompression() bool { return s.zipCompression } +func (s *appSettings) IndexPageEnabled() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.returnIndexPage +} + +func (s *appSettings) IndexPageTemplate() string { + s.mu.RLock() + defer s.mu.RUnlock() + if s.indexPageTemplate == "" { + return templates.DefaultIndexTemplate + } + return s.indexPageTemplate +} + func (s *appSettings) setZipCompression(val bool) { s.mu.Lock() s.zipCompression = val s.mu.Unlock() } +func (s *appSettings) setReturnIndexPage(val bool) { + s.mu.Lock() + s.returnIndexPage = val + s.mu.Unlock() +} + +func (s *appSettings) setIndexTemplate(val string) { + s.mu.Lock() + s.indexPageTemplate = val + s.mu.Unlock() +} + +func (a *app) loadIndexPageTemplate() { + if !a.settings.IndexPageEnabled() { + return + } + reader, err := os.Open(a.cfg.GetString(cfgIndexPageTemplatePath)) + if err != nil { + a.settings.setIndexTemplate("") + a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + return + } + tmpl, err := io.ReadAll(reader) + if err != nil { + a.settings.setIndexTemplate("") + a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + return + } + a.settings.setIndexTemplate(string(tmpl)) + a.log.Info(logs.SetCustomIndexPageTemplate) +} + func (s *appSettings) ClientCut() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -491,6 +543,7 @@ func (a *app) configReload(ctx context.Context) { a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled)) a.initTracing(ctx) + a.loadIndexPageTemplate() a.setHealthStatus() a.log.Info(logs.SIGHUPConfigReloadCompleted) @@ -499,6 +552,7 @@ func (a *app) configReload(ctx context.Context) { func (a *app) updateSettings() { a.settings.setDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) a.settings.setZipCompression(a.cfg.GetBool(cfgZipCompression)) + a.settings.setReturnIndexPage(a.cfg.GetBool(cfgIndexPageEnabled)) a.settings.setClientCut(a.cfg.GetBool(cfgClientCut)) a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut)) a.settings.setNamespaceHeader(a.cfg.GetString(cfgResolveNamespaceHeader)) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index d9bbc53..eab5b6b 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -62,6 +62,9 @@ const ( cfgReconnectInterval = "reconnect_interval" + cfgIndexPageEnabled = "index_page.enabled" + cfgIndexPageTemplatePath = "index_page.template_path" + // Web. cfgWebReadBufferSize = "web.read_buffer_size" cfgWebWriteBufferSize = "web.write_buffer_size" @@ -203,6 +206,9 @@ func settings() *viper.Viper { // pool: v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) + v.SetDefault(cfgIndexPageEnabled, false) + v.SetDefault(cfgIndexPageTemplatePath, "") + // frostfs: v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut) diff --git a/config/config.yaml b/config/config.yaml index 1b87fe9..61aa70b 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -107,6 +107,11 @@ request_timeout: 5s # Timeout to check node health during rebalance. rebalance_timer: 30s # Interval to check nodes health. pool_error_threshold: 100 # The number of errors on connection after which node is considered as unhealthy. +# Enable index page to see objects list for specified container and prefix +index_page: + enabled: false + template_path: internal/handler/templates/index.gotmpl + zip: compression: false # Enable zip compression to download files by common prefix. @@ -132,4 +137,4 @@ cache: resolve_bucket: namespace_header: X-Frostfs-Namespace - default_namespaces: [ "", "root" ] \ No newline at end of file + default_namespaces: [ "", "root" ] diff --git a/docs/api.md b/docs/api.md index 78df766..f7eb3a4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -95,12 +95,12 @@ The `filename` field from the multipart form will be set as `FileName` attribute ## Get object -Route: `/get/{cid}/{oid}?[download=true]` +Route: `/get/{cid}/{oid}?[download=false]` | Route parameter | Type | Description | |-----------------|--------|------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `cid` | Single | Base58 encoded container ID or container name from NNS. | -| `oid` | Single | Base58 encoded object ID. | +| `cid` | Single | Base58 encoded `container ID` or `container name` from NNS or `bucket name`. | +| `oid` | Single | Base58 encoded `object ID`. Also could be `S3 object name` if `cid` is specified as bucket name. | | `download` | Query | Set the `Content-Disposition` header as `attachment` in response.
This make the browser to download object as file instead of showing it on the page. | ### Methods @@ -141,6 +141,13 @@ Get an object (payload and attributes) by an address. | 400 | Some error occurred during object downloading. | | 404 | Container or object not found. | +###### Body + +Returns object data. If request performed from browser, either displays raw data or downloads it as +attachment if `download` query parameter is set to `true`. +If `index_page.enabled` is set to `true`, returns HTML with index-page if no object with specified +S3-name was found. + #### HEAD Get an object attributes by an address. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index fb5ad2f..e8d1f4b 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -57,6 +57,7 @@ $ cat http.log | `frostfs` | [Frostfs configuration](#frostfs-section) | | `cache` | [Cache configuration](#cache-section) | | `resolve_bucket` | [Bucket name resolving configuration](#resolve_bucket-section) | +| `index_page` | [Index page configuration](#index_page-section) | # General section @@ -75,16 +76,16 @@ pool_error_threshold: 100 reconnect_interval: 1m ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|------------------------|------------|---------------|----------------|------------------------------------------------------------------------------------| -| `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. | -| `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. | -| `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. | -| `stream_timeout` | `duration` | | `10s` | Timeout for individual operations in streaming RPC. | -| `request_timeout` | `duration` | | `15s` | Timeout to check node health during rebalance. | -| `rebalance_timer` | `duration` | | `60s` | Interval to check node health. | -| `pool_error_threshold` | `uint32` | | `100` | The number of errors on connection after which node is considered as unhealthy. | -| `reconnect_interval` | `duration` | no | `1m` | Listeners reconnection interval. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|------------------------|------------|---------------|---------------|-------------------------------------------------------------------------------------------------| +| `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. | +| `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. | +| `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. | +| `stream_timeout` | `duration` | | `10s` | Timeout for individual operations in streaming RPC. | +| `request_timeout` | `duration` | | `15s` | Timeout to check node health during rebalance. | +| `rebalance_timer` | `duration` | | `60s` | Interval to check node health. | +| `pool_error_threshold` | `uint32` | | `100` | The number of errors on connection after which node is considered as unhealthy. | +| `reconnect_interval` | `duration` | no | `1m` | Listeners reconnection interval. | # `wallet` section @@ -346,4 +347,19 @@ resolve_bucket: | Parameter | Type | SIGHUP reload | Default value | Description | |----------------------|------------|---------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------| | `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. | -| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | \ No newline at end of file +| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | + +# `index_page` section + +Parameters for index HTML-page output with S3-bucket or S3-subdir content for `Get object` request + +```yaml +index_page: + enabled: false + template_path: "" +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------------|----------|---------------|---------------|---------------------------------------------------------------------------------| +| `enabled` | `bool` | yes | `false` | Flag to enable index_page return if no object with specified S3-name was found. | +| `template_path` | `string` | yes | `""` | Path to .gotmpl file with html template for index_page. | diff --git a/go.mod b/go.mod index afa704b..d1a3788 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 + github.com/docker/go-units v0.4.0 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.106.2 github.com/prometheus/client_golang v1.19.0 @@ -50,7 +51,6 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/docker v20.10.14+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect - github.com/docker/go-units v0.4.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/internal/api/tree.go b/internal/api/tree.go index 4d16cc7..5b1d608 100644 --- a/internal/api/tree.go +++ b/internal/api/tree.go @@ -8,6 +8,7 @@ import ( type NodeVersion struct { BaseNodeVersion DeleteMarker bool + IsPrefixNode bool } // BaseNodeVersion is minimal node info from tree service. diff --git a/internal/frostfs/services/pool_wrapper.go b/internal/frostfs/services/pool_wrapper.go index f7b0a26..fa70f15 100644 --- a/internal/frostfs/services/pool_wrapper.go +++ b/internal/frostfs/services/pool_wrapper.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + "io" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" @@ -15,16 +17,16 @@ type GetNodeByPathResponseInfoWrapper struct { response *grpcService.GetNodeByPathResponse_Info } -func (n GetNodeByPathResponseInfoWrapper) GetNodeID() uint64 { - return n.response.GetNodeId() +func (n GetNodeByPathResponseInfoWrapper) GetNodeID() []uint64 { + return []uint64{n.response.GetNodeId()} } -func (n GetNodeByPathResponseInfoWrapper) GetParentID() uint64 { - return n.response.GetParentId() +func (n GetNodeByPathResponseInfoWrapper) GetParentID() []uint64 { + return []uint64{n.response.GetParentId()} } -func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() uint64 { - return n.response.GetTimestamp() +func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() []uint64 { + return []uint64{n.response.GetTimestamp()} } func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { @@ -89,3 +91,73 @@ func handleError(err error) error { return err } + +func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) { + order := treepool.NoneOrder + if sort { + order = treepool.AscendingOrder + } + poolPrm := treepool.GetSubTreeParams{ + CID: bktInfo.CID, + TreeID: treeID, + RootID: rootID, + Depth: depth, + BearerToken: getBearer(ctx), + Order: order, + } + if len(rootID) == 1 && rootID[0] == 0 { + // storage node interprets 'nil' value as []uint64{0} + // gate wants to send 'nil' value instead of []uint64{0}, because + // it provides compatibility with previous tree service api where + // single uint64(0) value is dropped from signature + poolPrm.RootID = nil + } + + subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) + if err != nil { + return nil, handleError(err) + } + + var subtree []tree.NodeResponse + + node, err := subTreeReader.Next() + for err == nil { + subtree = append(subtree, GetSubTreeResponseBodyWrapper{node}) + node, err = subTreeReader.Next() + } + if err != io.EOF { + return nil, handleError(err) + } + + return subtree, nil +} + +type GetSubTreeResponseBodyWrapper struct { + response *grpcService.GetSubTreeResponse_Body +} + +func (n GetSubTreeResponseBodyWrapper) GetNodeID() []uint64 { + return n.response.GetNodeId() +} + +func (n GetSubTreeResponseBodyWrapper) GetParentID() []uint64 { + resp := n.response.GetParentId() + if resp == nil { + // storage sends nil that should be interpreted as []uint64{0} + // due to protobuf compatibility, see 'GetSubTree' function + return []uint64{0} + } + return resp +} + +func (n GetSubTreeResponseBodyWrapper) GetTimestamp() []uint64 { + return n.response.GetTimestamp() +} + +func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta { + res := make([]tree.Meta, len(n.response.Meta)) + for i, value := range n.response.Meta { + res[i] = value + } + return res +} diff --git a/internal/handler/browse.go b/internal/handler/browse.go new file mode 100644 index 0000000..e84fb04 --- /dev/null +++ b/internal/handler/browse.go @@ -0,0 +1,157 @@ +package handler + +import ( + "html/template" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "github.com/docker/go-units" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +const ( + dateFormat = "02-01-2006 15:04" + attrOID = "OID" + attrCreated = "Created" + attrFileName = "FileName" + attrSize = "Size" +) + +type ( + BrowsePageData struct { + BucketName, + Prefix string + Objects []ResponseObject + } + ResponseObject struct { + OID string + Created string + FileName string + Size string + IsDir bool + } +) + +func parseTimestamp(tstamp string) (time.Time, error) { + millis, err := strconv.ParseInt(tstamp, 10, 64) + if err != nil { + return time.Time{}, err + } + + return time.UnixMilli(millis), nil +} + +func NewResponseObject(nodes map[string]string) ResponseObject { + return ResponseObject{ + OID: nodes[attrOID], + Created: nodes[attrCreated], + FileName: nodes[attrFileName], + Size: nodes[attrSize], + IsDir: nodes[attrOID] == "", + } +} + +func formatTimestamp(strdate string) string { + date, err := parseTimestamp(strdate) + if err != nil || date.IsZero() { + return "" + } + + return date.Format(dateFormat) +} + +func formatSize(strsize string) string { + size, err := strconv.ParseFloat(strsize, 64) + if err != nil { + return "0B" + } + return units.HumanSize(size) +} + +func parentDir(prefix string) string { + index := strings.LastIndex(prefix, "/") + if index == -1 { + return prefix + } + return prefix[index:] +} + +func trimPrefix(encPrefix string) string { + prefix, err := url.PathUnescape(encPrefix) + if err != nil { + return "" + } + slashIndex := strings.LastIndex(prefix, "/") + if slashIndex == -1 { + return "" + } + return prefix[:slashIndex] +} + +func urlencode(prefix, filename string) string { + var res strings.Builder + path := filename + if prefix != "" { + path = strings.Join([]string{prefix, filename}, "/") + } + prefixParts := strings.Split(path, "/") + for _, prefixPart := range prefixParts { + prefixPart = "/" + url.PathEscape(prefixPart) + if prefixPart == "/." || prefixPart == "/.." { + prefixPart = url.PathEscape(prefixPart) + } + res.WriteString(prefixPart) + } + + return res.String() +} + +func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketInfo, prefix string) { + log := h.log.With(zap.String("bucket", bucketInfo.Name)) + ctx := utils.GetContextFromRequest(c) + nodes, err := h.listObjects(ctx, bucketInfo, prefix) + if err != nil { + logAndSendBucketError(c, log, err) + return + } + + respObjects := make([]ResponseObject, len(nodes)) + + for i, node := range nodes { + respObjects[i] = NewResponseObject(node) + } + + sort.Slice(respObjects, func(i, j int) bool { + if respObjects[i].IsDir == respObjects[j].IsDir { + return respObjects[i].FileName < respObjects[j].FileName + } + return respObjects[i].IsDir + }) + indexTemplate := h.config.IndexPageTemplate() + + tmpl, err := template.New("index").Funcs(template.FuncMap{ + "formatTimestamp": formatTimestamp, + "formatSize": formatSize, + "trimPrefix": trimPrefix, + "urlencode": urlencode, + "parentDir": parentDir, + }).Parse(indexTemplate) + if err != nil { + logAndSendBucketError(c, log, err) + return + } + if err = tmpl.Execute(c, &BrowsePageData{ + BucketName: bucketInfo.Name, + Prefix: prefix, + Objects: respObjects, + }); err != nil { + logAndSendBucketError(c, log, err) + return + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4de9d9a..8b07af0 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -30,6 +30,8 @@ type Config interface { DefaultTimestamp() bool ZipCompression() bool ClientCut() bool + IndexPageEnabled() bool + IndexPageTemplate() string BufferMaxSizeForPut() uint64 NamespaceHeader() string } @@ -208,41 +210,50 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ // byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { +func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { var ( - bucketname = req.UserValue("cid").(string) - key = req.UserValue("oid").(string) + bucketname = c.UserValue("cid").(string) + key = c.UserValue("oid").(string) log = h.log.With(zap.String("bucketname", bucketname), zap.String("key", key)) + download = c.QueryArgs().GetBool("download") ) unescapedKey, err := url.QueryUnescape(key) if err != nil { - logAndSendBucketError(req, log, err) + logAndSendBucketError(c, log, err) return } - ctx := utils.GetContextFromRequest(req) + ctx := utils.GetContextFromRequest(c) bktInfo, err := h.getBucketInfo(ctx, bucketname, log) if err != nil { - logAndSendBucketError(req, log, err) + logAndSendBucketError(c, log, err) return } foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey) + if h.config.IndexPageEnabled() && !download && string(c.Method()) != fasthttp.MethodHead { + if isDir(unescapedKey) || isContainerRoot(unescapedKey) { + if code := checkErrorType(err); code == fasthttp.StatusNotFound || code == fasthttp.StatusOK { + c.SetStatusCode(code) + h.browseObjects(c, bktInfo, unescapedKey) + return + } + } + } if err != nil { if errors.Is(err, tree.ErrNodeAccessDenied) { - response.Error(req, "Access Denied", fasthttp.StatusForbidden) - return + response.Error(c, "Access Denied", fasthttp.StatusForbidden) + } else { + response.Error(c, "object wasn't found", fasthttp.StatusNotFound) + log.Error(logs.GetLatestObjectVersion, zap.Error(err)) } - log.Error(logs.GetLatestObjectVersion, zap.Error(err)) - - response.Error(req, "object wasn't found", fasthttp.StatusNotFound) return } if foundOid.DeleteMarker { log.Error(logs.ObjectWasDeleted) - response.Error(req, "object deleted", fasthttp.StatusNotFound) + response.Error(c, "object deleted", fasthttp.StatusNotFound) return } @@ -250,7 +261,7 @@ func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, addr.SetContainer(bktInfo.CID) addr.SetObject(foundOid.OID) - f(ctx, *h.newRequest(req, log), addr) + f(ctx, *h.newRequest(c, log), addr) } // byAttribute is a wrapper similar to byAddress. @@ -379,3 +390,25 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } + +func (h *Handler) listObjects(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]map[string]string, error) { + nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) + if err != nil { + return nil, err + } + + var objects = make([]map[string]string, 0, len(nodes)) + for _, node := range nodes { + meta := node.GetMeta() + if meta == nil { + continue + } + var obj = make(map[string]string, len(meta)) + for _, m := range meta { + obj[m.GetKey()] = string(m.GetValue()) + } + objects = append(objects, obj) + } + + return objects, nil +} diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index ed67f88..4fe9153 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -12,6 +12,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" @@ -37,6 +38,10 @@ func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree return nil, nil } +func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) { + return nil, nil +} + type configMock struct { } @@ -48,6 +53,17 @@ func (c *configMock) ZipCompression() bool { return false } +func (c *configMock) IndexPageEnabled() bool { + return false +} + +func (c *configMock) IndexPageTemplatePath() string { + return "" +} +func (c *configMock) IndexPageTemplate() string { + return "" +} + func (c *configMock) ClientCut() bool { return false } diff --git a/internal/handler/reader.go b/internal/handler/reader.go index 65d258b..d901f25 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -53,6 +53,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi dis = "inline" start = time.Now() filename string + filepath string ) prm := PrmObjectGet{ @@ -104,6 +105,8 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi time.Unix(value, 0).UTC().Format(http.TimeFormat)) case object.AttributeContentType: contentType = val + case object.AttributeFilePath: + filepath = val } } @@ -135,6 +138,10 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi } req.SetContentType(contentType) + if filename == "" { + filename = filepath + } + req.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) req.Response.SetBodyStream(rObj.Payload, int(payloadSize)) diff --git a/internal/handler/upload.go b/internal/handler/upload.go index cea2250..6c0e117 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -43,22 +43,22 @@ func (pr *putResponse) encode(w io.Writer) error { } // Upload handles multipart upload request. -func (h *Handler) Upload(req *fasthttp.RequestCtx) { +func (h *Handler) Upload(c *fasthttp.RequestCtx) { var ( file MultipartFile idObj oid.ID addr oid.Address - scid, _ = req.UserValue("cid").(string) + scid, _ = c.UserValue("cid").(string) log = h.log.With(zap.String("cid", scid)) - bodyStream = req.RequestBodyStream() + bodyStream = c.RequestBodyStream() drainBuf = make([]byte, drainBufSize) ) - ctx := utils.GetContextFromRequest(req) + ctx := utils.GetContextFromRequest(c) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { - logAndSendBucketError(req, log, err) + logAndSendBucketError(c, log, err) return } @@ -75,21 +75,21 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { zap.Error(err), ) }() - boundary := string(req.Request.Header.MultipartFormBoundary()) + boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(h.log, bodyStream, boundary); err != nil { log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) - response.Error(req, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) + response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } - filtered, err := filterHeaders(h.log, &req.Request.Header) + filtered, err := filterHeaders(h.log, &c.Request.Header) if err != nil { log.Error(logs.CouldNotProcessHeaders, zap.Error(err)) - response.Error(req, err.Error(), fasthttp.StatusBadRequest) + response.Error(c, err.Error(), fasthttp.StatusBadRequest) return } now := time.Now() - if rawHeader := req.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { + if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err)) } else { @@ -97,9 +97,9 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { } } - if err = utils.PrepareExpirationHeader(req, h.frostfs, filtered, now); err != nil { + if err = utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) - response.Error(req, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) + response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -143,7 +143,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { } if idObj, err = h.frostfs.CreateObject(ctx, prm); err != nil { - h.handlePutFrostFSErr(req, err) + h.handlePutFrostFSErr(c, err) return } @@ -151,9 +151,9 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { addr.SetContainer(bktInfo.CID) // Try to return the response, otherwise, if something went wrong, throw an error. - if err = newPutResponse(addr).encode(req); err != nil { + if err = newPutResponse(addr).encode(c); err != nil { log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) - response.Error(req, "could not encode response", fasthttp.StatusBadRequest) + response.Error(c, "could not encode response", fasthttp.StatusBadRequest) return } @@ -170,8 +170,8 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { } } // Report status code and content type. - req.Response.SetStatusCode(fasthttp.StatusOK) - req.Response.Header.SetContentType(jsonHeader) + c.Response.SetStatusCode(fasthttp.StatusOK) + c.Response.Header.SetContentType(jsonHeader) } func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) { diff --git a/internal/handler/utils.go b/internal/handler/utils.go index a5a53ed..7fa1158 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -2,12 +2,14 @@ package handler import ( "context" + "errors" "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/valyala/fasthttp" @@ -38,6 +40,25 @@ func bearerToken(ctx context.Context) *bearer.Token { return nil } +func isDir(name string) bool { + return strings.HasSuffix(name, "/") +} + +func isContainerRoot(key string) bool { + return key == "" +} + +func checkErrorType(err error) int { + switch { + case err == nil: + return fasthttp.StatusOK + case errors.Is(err, tree.ErrNodeAccessDenied): + return fasthttp.StatusForbidden + default: + return fasthttp.StatusNotFound + } +} + func isValidToken(s string) bool { for _, c := range s { if c <= ' ' || c > 127 { diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 0ab5dbf..96bdaa5 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -31,6 +31,8 @@ const ( CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go + FailedToReadIndexPageTemplate = "failed to read index page template, set default" // Warn in ../../app.go + SetCustomIndexPageTemplate = "set custom index page template" // Info in ../../app.go ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" // Info in ../../app.go diff --git a/internal/templates/index.gotmpl b/internal/templates/index.gotmpl new file mode 100644 index 0000000..ea66a62 --- /dev/null +++ b/internal/templates/index.gotmpl @@ -0,0 +1,90 @@ +{{$bucketName := .BucketName}} +{{ $prefix := trimPrefix .Prefix }} + + + + + Index of s3://{{$bucketName}}/{{if $prefix}}/{{$prefix}}/{{end}} + + + +

Index of s3://{{$bucketName}}/{{if $prefix}}{{$prefix}}/{{end}}

+ + + + + + + + + + + {{ $trimmedPrefix := trimPrefix $prefix }} + {{if $trimmedPrefix }} + + + + + + + {{else}} + + + + + + + {{end}} + {{range .Objects}} + + + + + + + {{end}} + +
FilenameSizeCreatedDownload
+ ⮐.. +
+ ⮐.. +
+ {{if .IsDir}} + 🗀 + + {{.FileName}}/ + + {{else}} + 🗎 + + {{.FileName}} + + {{end}} + {{if not .IsDir}}{{ formatSize .Size }}{{end}}{{if not .IsDir}}{{ formatTimestamp .Created }}{{end}} + {{ if not .IsDir }} + + Link + + {{ end }} +
+ + diff --git a/internal/templates/template.go b/internal/templates/template.go new file mode 100644 index 0000000..b9885e6 --- /dev/null +++ b/internal/templates/template.go @@ -0,0 +1,6 @@ +package templates + +import _ "embed" + +//go:embed index.gotmpl +var DefaultIndexTemplate string diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index b01860d..880a100 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -52,8 +52,8 @@ func BearerTokenFromCookie(h *fasthttp.RequestHeader) []byte { // StoreBearerTokenAppCtx extracts a bearer token from the header or cookie and stores // it in the application context. -func StoreBearerTokenAppCtx(ctx context.Context, req *fasthttp.RequestCtx) (context.Context, error) { - tkn, err := fetchBearerToken(req) +func StoreBearerTokenAppCtx(ctx context.Context, c *fasthttp.RequestCtx) (context.Context, error) { + tkn, err := fetchBearerToken(c) if err != nil { return nil, err } diff --git a/tree/tree.go b/tree/tree.go index a9135eb..162f41f 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -2,11 +2,13 @@ package tree import ( "context" + "errors" "fmt" "strings" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -20,6 +22,7 @@ type ( // Each method must return ErrNodeNotFound or ErrNodeAccessDenied if relevant. ServiceClient interface { GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error) + GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]NodeResponse, error) } treeNode struct { @@ -29,6 +32,7 @@ type ( GetNodesParams struct { CnrID cid.ID + BktInfo *data.BucketInfo TreeID string Path []string Meta []string @@ -54,6 +58,7 @@ const ( // keys for delete marker nodes. isDeleteMarkerKV = "IsDeleteMarker" + sizeKV = "Size" // versionTree -- ID of a tree with object versions. versionTree = "version" @@ -73,26 +78,28 @@ type Meta interface { type NodeResponse interface { GetMeta() []Meta - GetTimestamp() uint64 + GetTimestamp() []uint64 + GetNodeID() []uint64 + GetParentID() []uint64 } func newTreeNode(nodeInfo NodeResponse) (*treeNode, error) { - treeNode := &treeNode{ + tNode := &treeNode{ Meta: make(map[string]string, len(nodeInfo.GetMeta())), } for _, kv := range nodeInfo.GetMeta() { switch kv.GetKey() { case oidKV: - if err := treeNode.ObjID.DecodeString(string(kv.GetValue())); err != nil { + if err := tNode.ObjID.DecodeString(string(kv.GetValue())); err != nil { return nil, err } default: - treeNode.Meta[kv.GetKey()] = string(kv.GetValue()) + tNode.Meta[kv.GetKey()] = string(kv.GetValue()) } } - return treeNode, nil + return tNode, nil } func (n *treeNode) Get(key string) (string, bool) { @@ -106,29 +113,44 @@ func (n *treeNode) FileName() (string, bool) { } func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { - treeNode, err := newTreeNode(node) + tNode, err := newTreeNode(node) if err != nil { return nil, fmt.Errorf("invalid tree node: %w", err) } - return newNodeVersionFromTreeNode(treeNode), nil + return newNodeVersionFromTreeNode(tNode), nil } func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) - + size, _ := treeNode.Get(sizeKV) version := &api.NodeVersion{ BaseNodeVersion: api.BaseNodeVersion{ OID: treeNode.ObjID, }, DeleteMarker: isDeleteMarker, + IsPrefixNode: size == "", } return version } func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { - meta := []string{oidKV, isDeleteMarkerKV} + nodes, err := c.GetVersions(ctx, cnrID, objectName) + if err != nil { + return nil, err + } + + latestNode, err := getLatestVersionNode(nodes) + if err != nil { + return nil, err + } + + return newNodeVersion(latestNode) +} + +func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]NodeResponse, error) { + meta := []string{oidKV, isDeleteMarkerKV, sizeKV} path := pathFromName(objectName) p := &GetNodesParams{ @@ -139,30 +161,24 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s LatestOnly: false, AllAttrs: false, } - nodes, err := c.service.GetNodes(ctx, p) - if err != nil { - return nil, err - } - latestNode, err := getLatestNode(nodes) - if err != nil { - return nil, err - } - - return newNodeVersion(latestNode) + return c.service.GetNodes(ctx, p) } -func getLatestNode(nodes []NodeResponse) (NodeResponse, error) { +func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { var ( maxCreationTime uint64 targetIndexNode = -1 ) for i, node := range nodes { - currentCreationTime := node.GetTimestamp() - if checkExistOID(node.GetMeta()) && currentCreationTime > maxCreationTime { - maxCreationTime = currentCreationTime + if !checkExistOID(node.GetMeta()) { + continue + } + + if currentCreationTime := getMaxTimestamp(node); currentCreationTime > maxCreationTime { targetIndexNode = i + maxCreationTime = currentCreationTime } } @@ -187,3 +203,145 @@ func checkExistOID(meta []Meta) bool { func pathFromName(objectName string) []string { return strings.Split(objectName, separator) } + +func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]NodeResponse, string, error) { + rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix) + if err != nil { + return nil, "", err + } + subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false) + if err != nil { + if errors.Is(err, layer.ErrNodeNotFound) { + return nil, "", nil + } + return nil, "", err + } + + nodesMap := make(map[string][]NodeResponse, len(subTree)) + for _, node := range subTree { + if MultiID(rootID).Equal(node.GetNodeID()) { + continue + } + + fileName := GetFilename(node) + if !strings.HasPrefix(fileName, tailPrefix) { + continue + } + + nodes := nodesMap[fileName] + + // Add all nodes if flag latestOnly is false. + // Add all intermediate nodes + // and only latest leaf (object) nodes. To do this store and replace last leaf (object) node in nodes[0] + if len(nodes) == 0 { + nodes = []NodeResponse{node} + } else if !latestOnly || isIntermediate(node) { + nodes = append(nodes, node) + } else if isIntermediate(nodes[0]) { + nodes = append([]NodeResponse{node}, nodes...) + } else if getMaxTimestamp(node) > getMaxTimestamp(nodes[0]) { + nodes[0] = node + } + + nodesMap[fileName] = nodes + } + + result := make([]NodeResponse, 0, len(subTree)) + for _, nodes := range nodesMap { + result = append(result, nodes...) + } + + return result, strings.TrimSuffix(prefix, tailPrefix), nil +} + +func (c *Tree) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) ([]uint64, string, error) { + rootID := []uint64{0} + path := strings.Split(prefix, separator) + tailPrefix := path[len(path)-1] + + if len(path) > 1 { + var err error + rootID, err = c.getPrefixNodeID(ctx, bktInfo, treeID, path[:len(path)-1]) + if err != nil { + return nil, "", err + } + } + + return rootID, tailPrefix, nil +} + +func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefixPath []string) ([]uint64, error) { + p := &GetNodesParams{ + CnrID: bktInfo.CID, + BktInfo: bktInfo, + TreeID: treeID, + Path: prefixPath, + LatestOnly: false, + AllAttrs: true, + } + nodes, err := c.service.GetNodes(ctx, p) + if err != nil { + return nil, err + } + + var intermediateNodes []uint64 + for _, node := range nodes { + if isIntermediate(node) { + intermediateNodes = append(intermediateNodes, node.GetNodeID()...) + } + } + + if len(intermediateNodes) == 0 { + return nil, layer.ErrNodeNotFound + } + + return intermediateNodes, nil +} + +func GetFilename(node NodeResponse) string { + for _, kv := range node.GetMeta() { + if kv.GetKey() == FileNameKey { + return string(kv.GetValue()) + } + } + + return "" +} + +func isIntermediate(node NodeResponse) bool { + if len(node.GetMeta()) != 1 { + return false + } + + return node.GetMeta()[0].GetKey() == FileNameKey +} + +func getMaxTimestamp(node NodeResponse) uint64 { + var maxTimestamp uint64 + + for _, timestamp := range node.GetTimestamp() { + if timestamp > maxTimestamp { + maxTimestamp = timestamp + } + } + + return maxTimestamp +} + +type MultiID []uint64 + +func (m MultiID) Equal(id MultiID) bool { + seen := make(map[uint64]struct{}, len(m)) + + for i := range m { + seen[m[i]] = struct{}{} + } + + for i := range id { + if _, ok := seen[id[i]]; !ok { + return false + } + } + + return true +} diff --git a/tree/tree_test.go b/tree/tree_test.go index 7cd2314..69ac5f6 100644 --- a/tree/tree_test.go +++ b/tree/tree_test.go @@ -24,8 +24,8 @@ type nodeResponse struct { timestamp uint64 } -func (n nodeResponse) GetTimestamp() uint64 { - return n.timestamp +func (n nodeResponse) GetTimestamp() []uint64 { + return []uint64{n.timestamp} } func (n nodeResponse) GetMeta() []Meta { @@ -36,6 +36,13 @@ func (n nodeResponse) GetMeta() []Meta { return res } +func (n nodeResponse) GetNodeID() []uint64 { + return nil +} +func (n nodeResponse) GetParentID() []uint64 { + return nil +} + func TestGetLatestNode(t *testing.T) { for _, tc := range []struct { name string @@ -130,7 +137,7 @@ func TestGetLatestNode(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - actualNode, err := getLatestNode(tc.nodes) + actualNode, err := getLatestVersionNode(tc.nodes) if tc.error { require.Error(t, err) return From 495f7455355c11d3f4262ea61044a22bd45c301a Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Wed, 18 Sep 2024 07:35:26 +0300 Subject: [PATCH 473/548] [#142] Fix multipart-objects download Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 5 +- internal/handler/frostfs_mock.go | 11 +- internal/handler/handler.go | 19 +- internal/handler/multipart.go | 31 +++ internal/handler/reader.go | 98 ++++--- internal/handler/utils.go | 9 + internal/{ => service}/frostfs/frostfs.go | 4 +- .../service/frostfs/multi_object_reader.go | 241 ++++++++++++++++++ .../frostfs/multi_object_reader_test.go | 137 ++++++++++ .../frostfs}/pool_wrapper.go | 2 +- tree/tree_test.go | 22 +- 11 files changed, 517 insertions(+), 62 deletions(-) rename internal/{ => service}/frostfs/frostfs.go (97%) create mode 100644 internal/service/frostfs/multi_object_reader.go create mode 100644 internal/service/frostfs/multi_object_reader_test.go rename internal/{frostfs/services => service/frostfs}/pool_wrapper.go (99%) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 8d5930d..f8300ec 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -19,11 +19,10 @@ import ( v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" @@ -453,7 +452,7 @@ func (a *app) setHealthStatus() { } func (a *app) Serve() { - handler := handler.New(a.AppParams(), a.settings, tree.NewTree(services.NewPoolWrapper(a.treePool))) + handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool))) // Configure router. a.configureRouter(handler) diff --git a/internal/handler/frostfs_mock.go b/internal/handler/frostfs_mock.go index 9f4378a..b60915e 100644 --- a/internal/handler/frostfs_mock.go +++ b/internal/handler/frostfs_mock.go @@ -229,6 +229,10 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) (Res return &resObjectSearchMock{res: res}, nil } +func (t *TestFrostFS) InitMultiObjectReader(context.Context, PrmInitMultiObjectReader) (io.Reader, error) { + return nil, nil +} + func isMatched(attributes []object.Attribute, filter object.SearchFilter) bool { for _, attr := range attributes { if attr.Key() == filter.Header() { @@ -269,10 +273,3 @@ func (t *TestFrostFS) isAllowed(cnrID cid.ID, userID user.ID, op acl.Op, objID o } return false } - -func newAddress(cnr cid.ID, obj oid.ID) oid.Address { - var addr oid.Address - addr.SetContainer(cnr) - addr.SetObject(obj) - return addr -} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 8b07af0..c680706 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -119,6 +119,14 @@ type PrmObjectSearch struct { Filters object.SearchFilters } +type PrmInitMultiObjectReader struct { + // payload range + Off, Ln uint64 + + Addr oid.Address + Bearer *bearer.Token +} + type ResObjectSearch interface { Read(buf []oid.ID) (int, error) Iterate(f func(oid.ID) bool) error @@ -140,6 +148,8 @@ type FrostFS interface { RangeObject(context.Context, PrmObjectRange) (io.ReadCloser, error) CreateObject(context.Context, PrmObjectCreate) (oid.ID, error) SearchObjects(context.Context, PrmObjectSearch) (ResObjectSearch, error) + InitMultiObjectReader(ctx context.Context, p PrmInitMultiObjectReader) (io.Reader, error) + utils.EpochInfoFetcher } @@ -201,9 +211,7 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ return } - var addr oid.Address - addr.SetContainer(bktInfo.CID) - addr.SetObject(*objID) + addr := newAddress(bktInfo.CID, *objID) f(ctx, *h.newRequest(c, log), addr) } @@ -256,10 +264,7 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r response.Error(c, "object deleted", fasthttp.StatusNotFound) return } - - var addr oid.Address - addr.SetContainer(bktInfo.CID) - addr.SetObject(foundOid.OID) + addr := newAddress(bktInfo.CID, foundOid.OID) f(ctx, *h.newRequest(c, log), addr) } diff --git a/internal/handler/multipart.go b/internal/handler/multipart.go index de9242f..213286c 100644 --- a/internal/handler/multipart.go +++ b/internal/handler/multipart.go @@ -1,13 +1,17 @@ package handler import ( + "errors" "io" + "strconv" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/multipart" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "go.uber.org/zap" ) +const attributeMultipartObjectSize = "S3-Multipart-Object-Size" + // MultipartFile provides standard ReadCloser interface and also allows one to // get file name, it's used for multipart uploads. type MultipartFile interface { @@ -45,3 +49,30 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF return part, nil } } + +// getPayload returns initial payload if object is not multipart else composes new reader with parts data. +func (h *Handler) getPayload(p getMultiobjectBodyParams) (io.ReadCloser, uint64, error) { + cid, ok := p.obj.Header.ContainerID() + if !ok { + return nil, 0, errors.New("no container id set") + } + oid, ok := p.obj.Header.ID() + if !ok { + return nil, 0, errors.New("no object id set") + } + size, err := strconv.ParseUint(p.strSize, 10, 64) + if err != nil { + return nil, 0, err + } + ctx := p.req.RequestCtx + params := PrmInitMultiObjectReader{ + Addr: newAddress(cid, oid), + Bearer: bearerToken(ctx), + } + payload, err := h.frostfs.InitMultiObjectReader(ctx, params) + if err != nil { + return nil, 0, err + } + + return io.NopCloser(payload), size, nil +} diff --git a/internal/handler/reader.go b/internal/handler/reader.go index d901f25..50121c9 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -47,20 +47,26 @@ func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (str return http.DetectContentType(buf), buf, err // to not lose io.EOF } -func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oid.Address) { +type getMultiobjectBodyParams struct { + obj *Object + req request + strSize string +} + +func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.Address) { var ( - err error - dis = "inline" - start = time.Now() - filename string - filepath string + shouldDownload = req.QueryArgs().GetBool("download") + start = time.Now() + filename string + filepath string + contentType string ) prm := PrmObjectGet{ PrmAuth: PrmAuth{ BearerToken: bearerToken(ctx), }, - Address: objectAddress, + Address: objAddress, } rObj, err := h.frostfs.GetObject(ctx, prm) @@ -70,15 +76,9 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi } // we can't close reader in this function, so how to do it? - - if req.Request.URI().QueryArgs().GetBool("download") { - dis = "attachment" - } - + req.setIDs(rObj.Header) + payload := rObj.Payload payloadSize := rObj.Header.PayloadSize() - - req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) - var contentType string for _, attr := range rObj.Header.Attributes() { key := attr.Key() val := attr.Value() @@ -93,31 +93,41 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi case object.AttributeFileName: filename = val case object.AttributeTimestamp: - value, err := strconv.ParseInt(val, 10, 64) - if err != nil { - req.log.Info(logs.CouldntParseCreationDate, - zap.String("key", key), + if err = req.setTimestamp(val); err != nil { + req.log.Error(logs.CouldntParseCreationDate, zap.String("val", val), zap.Error(err)) - continue } - req.Response.Header.Set(fasthttp.HeaderLastModified, - time.Unix(value, 0).UTC().Format(http.TimeFormat)) case object.AttributeContentType: contentType = val case object.AttributeFilePath: filepath = val + case attributeMultipartObjectSize: + payload, payloadSize, err = h.getPayload(getMultiobjectBodyParams{ + obj: rObj, + req: req, + strSize: val, + }) + if err != nil { + req.handleFrostFSErr(err, start) + return + } } } + if filename == "" { + filename = filepath + } - idsToResponse(&req.Response, &rObj.Header) + req.setDisposition(shouldDownload, filename) + + req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) if len(contentType) == 0 { // determine the Content-Type from the payload head var payloadHead []byte contentType, payloadHead, err = readContentType(payloadSize, func(uint64) (io.Reader, error) { - return rObj.Payload, nil + return payload, nil }) if err != nil && err != io.EOF { req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) @@ -129,20 +139,46 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oi var headReader io.Reader = bytes.NewReader(payloadHead) if err != io.EOF { // otherwise, we've already read full payload - headReader = io.MultiReader(headReader, rObj.Payload) + headReader = io.MultiReader(headReader, payload) } // note: we could do with io.Reader, but SetBodyStream below closes body stream // if it implements io.Closer and that's useful for us. - rObj.Payload = readCloser{headReader, rObj.Payload} + payload = readCloser{headReader, payload} } req.SetContentType(contentType) + req.Response.SetBodyStream(payload, int(payloadSize)) +} - if filename == "" { - filename = filepath +func (r *request) setIDs(obj object.Object) { + objID, _ := obj.ID() + cnrID, _ := obj.ContainerID() + r.Response.Header.Set(hdrObjectID, objID.String()) + r.Response.Header.Set(hdrOwnerID, obj.OwnerID().String()) + r.Response.Header.Set(hdrContainerID, cnrID.String()) +} + +func (r *request) setDisposition(shouldDownload bool, filename string) { + const ( + inlineDisposition = "inline" + attachmentDisposition = "attachment" + ) + + dis := inlineDisposition + if shouldDownload { + dis = attachmentDisposition } - req.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) - - req.Response.SetBodyStream(rObj.Payload, int(payloadSize)) + r.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) +} + +func (r *request) setTimestamp(timestamp string) error { + value, err := strconv.ParseInt(timestamp, 10, 64) + if err != nil { + return err + } + r.Response.Header.Set(fasthttp.HeaderLastModified, + time.Unix(value, 0).UTC().Format(http.TimeFormat)) + + return nil } diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 7fa1158..a944b67 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -12,6 +12,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -90,3 +92,10 @@ func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { } response.Error(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) } + +func newAddress(cnr cid.ID, obj oid.ID) oid.Address { + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(obj) + return addr +} diff --git a/internal/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go similarity index 97% rename from internal/frostfs/frostfs.go rename to internal/service/frostfs/frostfs.go index 6cf162a..c7e56a4 100644 --- a/internal/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -33,9 +33,9 @@ func NewFrostFS(p *pool.Pool) *FrostFS { } // Container implements frostfs.FrostFS interface method. -func (x *FrostFS) Container(ctx context.Context, layerPrm handler.PrmContainer) (*container.Container, error) { +func (x *FrostFS) Container(ctx context.Context, containerPrm handler.PrmContainer) (*container.Container, error) { prm := pool.PrmContainerGet{ - ContainerID: layerPrm.ContainerID, + ContainerID: containerPrm.ContainerID, } res, err := x.pool.GetContainer(ctx, prm) diff --git a/internal/service/frostfs/multi_object_reader.go b/internal/service/frostfs/multi_object_reader.go new file mode 100644 index 0000000..93f1f60 --- /dev/null +++ b/internal/service/frostfs/multi_object_reader.go @@ -0,0 +1,241 @@ +package frostfs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +// PartInfo is upload information about part. +type PartInfo struct { + Key string `json:"key"` + UploadID string `json:"uploadId"` + Number int `json:"number"` + OID oid.ID `json:"oid"` + Size uint64 `json:"size"` + ETag string `json:"etag"` + MD5 string `json:"md5"` + Created time.Time `json:"created"` +} + +type GetFrostFSParams struct { + // payload range + Off, Ln uint64 + Addr oid.Address +} + +type PartObj struct { + OID oid.ID + Size uint64 +} + +type readerInitiator interface { + InitFrostFSObjectPayloadReader(ctx context.Context, p GetFrostFSParams) (io.ReadCloser, error) +} + +// MultiObjectReader implements io.Reader of payloads of the object list stored in the FrostFS network. +type MultiObjectReader struct { + ctx context.Context + + layer readerInitiator + + startPartOffset uint64 + endPartLength uint64 + + prm GetFrostFSParams + + curIndex int + curReader io.ReadCloser + + parts []PartObj +} + +type MultiObjectReaderConfig struct { + Initiator readerInitiator + + // the offset of complete object and total size to read + Off, Ln uint64 + + Addr oid.Address + Parts []PartObj +} + +var ( + errOffsetIsOutOfRange = errors.New("offset is out of payload range") + errLengthIsOutOfRange = errors.New("length is out of payload range") + errEmptyPartsList = errors.New("empty parts list") + errorZeroRangeLength = errors.New("zero range length") +) + +func (x *FrostFS) InitMultiObjectReader(ctx context.Context, p handler.PrmInitMultiObjectReader) (io.Reader, error) { + combinedObj, err := x.GetObject(ctx, handler.PrmObjectGet{ + PrmAuth: handler.PrmAuth{BearerToken: p.Bearer}, + Address: p.Addr, + }) + if err != nil { + return nil, fmt.Errorf("get combined object '%s': %w", p.Addr.Object().EncodeToString(), err) + } + + var parts []*PartInfo + if err = json.NewDecoder(combinedObj.Payload).Decode(&parts); err != nil { + return nil, fmt.Errorf("unmarshal combined object parts: %w", err) + } + + objParts := make([]PartObj, len(parts)) + for i, part := range parts { + objParts[i] = PartObj{ + OID: part.OID, + Size: part.Size, + } + } + + return NewMultiObjectReader(ctx, MultiObjectReaderConfig{ + Initiator: x, + Off: p.Off, + Ln: p.Ln, + Parts: objParts, + Addr: p.Addr, + }) +} + +func NewMultiObjectReader(ctx context.Context, cfg MultiObjectReaderConfig) (*MultiObjectReader, error) { + if len(cfg.Parts) == 0 { + return nil, errEmptyPartsList + } + + r := &MultiObjectReader{ + ctx: ctx, + layer: cfg.Initiator, + prm: GetFrostFSParams{ + Addr: cfg.Addr, + }, + parts: cfg.Parts, + } + + if cfg.Off+cfg.Ln == 0 { + return r, nil + } + + if cfg.Off > 0 && cfg.Ln == 0 { + return nil, errorZeroRangeLength + } + + startPartIndex, startPartOffset := findStartPart(cfg) + if startPartIndex == -1 { + return nil, errOffsetIsOutOfRange + } + r.startPartOffset = startPartOffset + + endPartIndex, endPartLength := findEndPart(cfg) + if endPartIndex == -1 { + return nil, errLengthIsOutOfRange + } + r.endPartLength = endPartLength + + r.parts = cfg.Parts[startPartIndex : endPartIndex+1] + + return r, nil +} + +func findStartPart(cfg MultiObjectReaderConfig) (index int, offset uint64) { + position := cfg.Off + for i, part := range cfg.Parts { + // Strict inequality when searching for start position to avoid reading zero length part. + if position < part.Size { + return i, position + } + position -= part.Size + } + + return -1, 0 +} + +func findEndPart(cfg MultiObjectReaderConfig) (index int, length uint64) { + position := cfg.Off + cfg.Ln + for i, part := range cfg.Parts { + // Non-strict inequality when searching for end position to avoid out of payload range error. + if position <= part.Size { + return i, position + } + position -= part.Size + } + + return -1, 0 +} + +func (x *MultiObjectReader) Read(p []byte) (n int, err error) { + if x.curReader != nil { + n, err = x.curReader.Read(p) + if err != nil { + if closeErr := x.curReader.Close(); closeErr != nil { + return n, fmt.Errorf("%w (close err: %v)", err, closeErr) + } + } + if !errors.Is(err, io.EOF) { + return n, err + } + + x.curIndex++ + } + + if x.curIndex == len(x.parts) { + return n, io.EOF + } + + x.prm.Addr.SetObject(x.parts[x.curIndex].OID) + + if x.curIndex == 0 { + x.prm.Off = x.startPartOffset + x.prm.Ln = x.parts[x.curIndex].Size - x.startPartOffset + } + + if x.curIndex == len(x.parts)-1 { + x.prm.Ln = x.endPartLength - x.prm.Off + } + + x.curReader, err = x.layer.InitFrostFSObjectPayloadReader(x.ctx, x.prm) + if err != nil { + return n, fmt.Errorf("init payload reader for the next part: %w", err) + } + + x.prm.Off = 0 + x.prm.Ln = 0 + + next, err := x.Read(p[n:]) + + return n + next, err +} + +// InitFrostFSObjectPayloadReader initializes payload reader of the FrostFS object. +// Zero range corresponds to full payload (panics if only offset is set). +func (x *FrostFS) InitFrostFSObjectPayloadReader(ctx context.Context, p GetFrostFSParams) (io.ReadCloser, error) { + var prmAuth handler.PrmAuth + + if p.Off+p.Ln != 0 { + prm := handler.PrmObjectRange{ + PrmAuth: prmAuth, + PayloadRange: [2]uint64{p.Off, p.Ln}, + Address: p.Addr, + } + + return x.RangeObject(ctx, prm) + } + + prm := handler.PrmObjectGet{ + PrmAuth: prmAuth, + Address: p.Addr, + } + + res, err := x.GetObject(ctx, prm) + if err != nil { + return nil, err + } + + return res.Payload, nil +} diff --git a/internal/service/frostfs/multi_object_reader_test.go b/internal/service/frostfs/multi_object_reader_test.go new file mode 100644 index 0000000..4127cdc --- /dev/null +++ b/internal/service/frostfs/multi_object_reader_test.go @@ -0,0 +1,137 @@ +package frostfs + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "testing" + + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +type readerInitiatorMock struct { + parts map[oid.ID][]byte +} + +func (r *readerInitiatorMock) InitFrostFSObjectPayloadReader(_ context.Context, p GetFrostFSParams) (io.ReadCloser, error) { + partPayload, ok := r.parts[p.Addr.Object()] + if !ok { + return nil, errors.New("part not found") + } + + if p.Off+p.Ln == 0 { + return io.NopCloser(bytes.NewReader(partPayload)), nil + } + + if p.Off > uint64(len(partPayload)-1) { + return nil, fmt.Errorf("invalid offset: %d/%d", p.Off, len(partPayload)) + } + + if p.Off+p.Ln > uint64(len(partPayload)) { + return nil, fmt.Errorf("invalid range: %d-%d/%d", p.Off, p.Off+p.Ln, len(partPayload)) + } + + return io.NopCloser(bytes.NewReader(partPayload[p.Off : p.Off+p.Ln])), nil +} + +func prepareDataReader() ([]byte, []PartObj, *readerInitiatorMock) { + mockInitReader := &readerInitiatorMock{ + parts: map[oid.ID][]byte{ + oidtest.ID(): []byte("first part 1"), + oidtest.ID(): []byte("second part 2"), + oidtest.ID(): []byte("third part 3"), + }, + } + + var fullPayload []byte + parts := make([]PartObj, 0, len(mockInitReader.parts)) + for id, payload := range mockInitReader.parts { + parts = append(parts, PartObj{OID: id, Size: uint64(len(payload))}) + fullPayload = append(fullPayload, payload...) + } + + return fullPayload, parts, mockInitReader +} + +func TestMultiReader(t *testing.T) { + ctx := context.Background() + + fullPayload, parts, mockInitReader := prepareDataReader() + + for _, tc := range []struct { + name string + off uint64 + ln uint64 + err error + }{ + { + name: "simple read all", + }, + { + name: "simple read with length", + ln: uint64(len(fullPayload)), + }, + { + name: "middle of parts", + off: parts[0].Size + 2, + ln: 4, + }, + { + name: "first and second", + off: parts[0].Size - 4, + ln: 8, + }, + { + name: "first and third", + off: parts[0].Size - 4, + ln: parts[1].Size + 8, + }, + { + name: "second part", + off: parts[0].Size, + ln: parts[1].Size, + }, + { + name: "second and third", + off: parts[0].Size, + ln: parts[1].Size + parts[2].Size, + }, + { + name: "offset out of range", + off: uint64(len(fullPayload) + 1), + ln: 1, + err: errOffsetIsOutOfRange, + }, + { + name: "zero length", + off: parts[1].Size + 1, + ln: 0, + err: errorZeroRangeLength, + }, + } { + t.Run(tc.name, func(t *testing.T) { + multiReader, err := NewMultiObjectReader(ctx, MultiObjectReaderConfig{ + Initiator: mockInitReader, + Parts: parts, + Off: tc.off, + Ln: tc.ln, + }) + require.ErrorIs(t, err, tc.err) + + if tc.err == nil { + off := tc.off + ln := tc.ln + if off+ln == 0 { + ln = uint64(len(fullPayload)) + } + data, err := io.ReadAll(multiReader) + require.NoError(t, err) + require.Equal(t, fullPayload[off:off+ln], data) + } + }) + } +} diff --git a/internal/frostfs/services/pool_wrapper.go b/internal/service/frostfs/pool_wrapper.go similarity index 99% rename from internal/frostfs/services/pool_wrapper.go rename to internal/service/frostfs/pool_wrapper.go index fa70f15..b978d73 100644 --- a/internal/frostfs/services/pool_wrapper.go +++ b/internal/service/frostfs/pool_wrapper.go @@ -1,4 +1,4 @@ -package services +package frostfs import ( "context" diff --git a/tree/tree_test.go b/tree/tree_test.go index 69ac5f6..62f9914 100644 --- a/tree/tree_test.go +++ b/tree/tree_test.go @@ -21,11 +21,11 @@ func (m nodeMeta) GetValue() []byte { type nodeResponse struct { meta []nodeMeta - timestamp uint64 + timestamp []uint64 } func (n nodeResponse) GetTimestamp() []uint64 { - return []uint64{n.timestamp} + return n.timestamp } func (n nodeResponse) GetMeta() []Meta { @@ -59,7 +59,7 @@ func TestGetLatestNode(t *testing.T) { name: "one node of the object version", nodes: []NodeResponse{ nodeResponse{ - timestamp: 1, + timestamp: []uint64{1}, meta: []nodeMeta{ { key: oidKV, @@ -74,11 +74,11 @@ func TestGetLatestNode(t *testing.T) { name: "one node of the object version and one node of the secondary object", nodes: []NodeResponse{ nodeResponse{ - timestamp: 3, + timestamp: []uint64{3}, meta: []nodeMeta{}, }, nodeResponse{ - timestamp: 1, + timestamp: []uint64{1}, meta: []nodeMeta{ { key: oidKV, @@ -93,11 +93,11 @@ func TestGetLatestNode(t *testing.T) { name: "all nodes represent a secondary object", nodes: []NodeResponse{ nodeResponse{ - timestamp: 3, + timestamp: []uint64{3}, meta: []nodeMeta{}, }, nodeResponse{ - timestamp: 5, + timestamp: []uint64{5}, meta: []nodeMeta{}, }, }, @@ -107,7 +107,7 @@ func TestGetLatestNode(t *testing.T) { name: "several nodes of different types and with different timestamp", nodes: []NodeResponse{ nodeResponse{ - timestamp: 1, + timestamp: []uint64{1}, meta: []nodeMeta{ { key: oidKV, @@ -116,11 +116,11 @@ func TestGetLatestNode(t *testing.T) { }, }, nodeResponse{ - timestamp: 3, + timestamp: []uint64{3}, meta: []nodeMeta{}, }, nodeResponse{ - timestamp: 4, + timestamp: []uint64{4}, meta: []nodeMeta{ { key: oidKV, @@ -129,7 +129,7 @@ func TestGetLatestNode(t *testing.T) { }, }, nodeResponse{ - timestamp: 6, + timestamp: []uint64{6}, meta: []nodeMeta{}, }, }, From fc86ab3511a153b30b7ff45b15ff5865405786ca Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Tue, 8 Oct 2024 12:08:39 +0300 Subject: [PATCH 474/548] [#148] Add trace_id to logs Signed-off-by: Roman Loginov --- CHANGELOG.md | 3 +++ cmd/http-gw/app.go | 41 ++++++++++++++++++++++++------------ internal/handler/browse.go | 4 +++- internal/handler/download.go | 9 ++++---- internal/handler/handler.go | 35 +++++++++++++++--------------- internal/handler/upload.go | 28 +++++++++++++----------- utils/util.go | 32 ++++++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a489027..46e9c23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,10 @@ This document outlines major changes between releases. ## [Unreleased] + +### Added - Support percent-encoding for GET queries (#134) +- Add `trace_id` to logs (#148) ### Changed - Update go version to 1.22 (#132) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index f8300ec..b26df89 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -42,6 +42,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/viper" "github.com/valyala/fasthttp" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "golang.org/x/exp/slices" ) @@ -589,15 +590,15 @@ func (a *app) configureRouter(handler *handler.Handler) { response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.Upload)))))) + r.POST("/upload/{cid}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.Upload)))))) a.log.Info(logs.AddedPathUploadCid) - r.GET("/get/{cid}/{oid:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadByAddressOrBucketName)))))) - r.HEAD("/get/{cid}/{oid:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.HeadByAddressOrBucketName)))))) + r.GET("/get/{cid}/{oid:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadByAddressOrBucketName)))))) + r.HEAD("/get/{cid}/{oid:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.HeadByAddressOrBucketName)))))) a.log.Info(logs.AddedPathGetCidOid) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadByAttribute)))))) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.HeadByAttribute)))))) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadByAttribute)))))) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.HeadByAttribute)))))) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.logger(a.canonicalizer(a.tokenizer(a.tracer(a.reqNamespace(handler.DownloadZipped)))))) + r.GET("/zip/{cid}/{prefix:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadZipped)))))) a.log.Info(logs.AddedPathZipCidPrefix) a.webServer.Handler = r.Handler @@ -605,11 +606,24 @@ func (a *app) configureRouter(handler *handler.Handler) { func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { - a.log.Info(logs.Request, zap.String("remote", req.RemoteAddr().String()), + requiredFields := []zap.Field{zap.Uint64("id", req.ID())} + reqCtx := utils.GetContextFromRequest(req) + if traceID := trace.SpanFromContext(reqCtx).SpanContext().TraceID(); traceID.IsValid() { + requiredFields = append(requiredFields, zap.String("trace_id", traceID.String())) + } + log := a.log.With(requiredFields...) + + reqCtx = utils.SetReqLog(reqCtx, log) + utils.SetContextToRequest(reqCtx, req) + + fields := []zap.Field{ + zap.String("remote", req.RemoteAddr().String()), zap.ByteString("method", req.Method()), zap.ByteString("path", req.Path()), zap.ByteString("query", req.QueryArgs().QueryString()), - zap.Uint64("id", req.ID())) + } + + log.Info(logs.Request, fields...) h(req) } } @@ -648,9 +662,12 @@ func (a *app) canonicalizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { - appCtx, err := tokens.StoreBearerTokenAppCtx(a.ctx, req) + reqCtx := utils.GetContextFromRequest(req) + appCtx, err := tokens.StoreBearerTokenAppCtx(reqCtx, req) if err != nil { - a.log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Uint64("id", req.ID()), zap.Error(err)) + log := utils.GetReqLogOrDefault(reqCtx, a.log) + + log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err)) response.Error(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -661,9 +678,7 @@ func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { func (a *app) tracer(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { - appCtx := utils.GetContextFromRequest(req) - - appCtx, span := utils.StartHTTPServerSpan(appCtx, req, "REQUEST") + appCtx, span := utils.StartHTTPServerSpan(a.ctx, req, "REQUEST") defer func() { utils.SetHTTPTraceInfo(appCtx, span, req) span.End() diff --git a/internal/handler/browse.go b/internal/handler/browse.go index e84fb04..c89d8c5 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -113,8 +113,10 @@ func urlencode(prefix, filename string) string { } func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketInfo, prefix string) { - log := h.log.With(zap.String("bucket", bucketInfo.Name)) ctx := utils.GetContextFromRequest(c) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("bucket", bucketInfo.Name)) + nodes, err := h.listObjects(ctx, bucketInfo, prefix) if err != nil { logAndSendBucketError(c, log, err) diff --git a/internal/handler/download.go b/internal/handler/download.go index 88109a6..19380d4 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -84,16 +84,17 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { scid, _ := c.UserValue("cid").(string) prefix, _ := c.UserValue("prefix").(string) + ctx := utils.GetContextFromRequest(c) + log := utils.GetReqLogOrDefault(ctx, h.log) + prefix, err := url.QueryUnescape(prefix) if err != nil { - h.log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Uint64("id", c.ID()), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) response.Error(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) return } - log := h.log.With(zap.String("cid", scid), zap.String("prefix", prefix), zap.Uint64("id", c.ID())) - - ctx := utils.GetContextFromRequest(c) + log = log.With(zap.String("cid", scid), zap.String("prefix", prefix)) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index c680706..62d0897 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -190,13 +190,12 @@ func New(params *AppParams, config Config, tree *tree.Tree) *Handler { // byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { - var ( - idCnr, _ = c.UserValue("cid").(string) - idObj, _ = c.UserValue("oid").(string) - log = h.log.With(zap.String("cid", idCnr), zap.String("oid", idObj)) - ) + idCnr, _ := c.UserValue("cid").(string) + idObj, _ := c.UserValue("oid").(string) ctx := utils.GetContextFromRequest(c) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("cid", idCnr), zap.String("oid", idObj)) bktInfo, err := h.getBucketInfo(ctx, idCnr, log) if err != nil { @@ -219,12 +218,13 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ // byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { - var ( - bucketname = c.UserValue("cid").(string) - key = c.UserValue("oid").(string) - log = h.log.With(zap.String("bucketname", bucketname), zap.String("key", key)) - download = c.QueryArgs().GetBool("download") - ) + bucketname := c.UserValue("cid").(string) + key := c.UserValue("oid").(string) + download := c.QueryArgs().GetBool("download") + + ctx := utils.GetContextFromRequest(c) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("bucketname", bucketname), zap.String("key", key)) unescapedKey, err := url.QueryUnescape(key) if err != nil { @@ -232,8 +232,6 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r return } - ctx := utils.GetContextFromRequest(c) - bktInfo, err := h.getBucketInfo(ctx, bucketname, log) if err != nil { logAndSendBucketError(c, log, err) @@ -275,23 +273,24 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re key, _ := c.UserValue("attr_key").(string) val, _ := c.UserValue("attr_val").(string) + ctx := utils.GetContextFromRequest(c) + log := utils.GetReqLogOrDefault(ctx, h.log) + key, err := url.QueryUnescape(key) if err != nil { - h.log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_key", key), zap.Uint64("id", c.ID()), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_key", key), zap.Error(err)) response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) return } val, err = url.QueryUnescape(val) if err != nil { - h.log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_val", val), zap.Uint64("id", c.ID()), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_val", val), zap.Error(err)) response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) return } - log := h.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) - - ctx := utils.GetContextFromRequest(c) + log = log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 6c0e117..867025d 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -45,16 +45,18 @@ func (pr *putResponse) encode(w io.Writer) error { // Upload handles multipart upload request. func (h *Handler) Upload(c *fasthttp.RequestCtx) { var ( - file MultipartFile - idObj oid.ID - addr oid.Address - scid, _ = c.UserValue("cid").(string) - log = h.log.With(zap.String("cid", scid)) - bodyStream = c.RequestBodyStream() - drainBuf = make([]byte, drainBufSize) + file MultipartFile + idObj oid.ID + addr oid.Address ) + scid, _ := c.UserValue("cid").(string) + bodyStream := c.RequestBodyStream() + drainBuf := make([]byte, drainBufSize) + ctx := utils.GetContextFromRequest(c) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("cid", scid)) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { @@ -75,13 +77,15 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { zap.Error(err), ) }() + boundary := string(c.Request.Header.MultipartFormBoundary()) - if file, err = fetchMultipartFile(h.log, bodyStream, boundary); err != nil { + if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } - filtered, err := filterHeaders(h.log, &c.Request.Header) + + filtered, err := filterHeaders(log, &c.Request.Header) if err != nil { log.Error(logs.CouldNotProcessHeaders, zap.Error(err)) response.Error(c, err.Error(), fasthttp.StatusBadRequest) @@ -143,7 +147,7 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { } if idObj, err = h.frostfs.CreateObject(ctx, prm); err != nil { - h.handlePutFrostFSErr(c, err) + h.handlePutFrostFSErr(c, err, log) return } @@ -174,11 +178,11 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { c.Response.Header.SetContentType(jsonHeader) } -func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) { +func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *zap.Logger) { statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err) logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) - h.log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) + log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) response.Error(r, msg, statusCode) } diff --git a/utils/util.go b/utils/util.go index d513817..b7f5e39 100644 --- a/utils/util.go +++ b/utils/util.go @@ -4,6 +4,7 @@ import ( "context" "github.com/valyala/fasthttp" + "go.uber.org/zap" ) // SetContextToRequest adds new context to fasthttp request. @@ -15,3 +16,34 @@ func SetContextToRequest(ctx context.Context, c *fasthttp.RequestCtx) { func GetContextFromRequest(c *fasthttp.RequestCtx) context.Context { return c.UserValue("context").(context.Context) } + +type ctxReqLoggerKeyType struct{} + +// SetReqLog sets child zap.Logger in the context. +func SetReqLog(ctx context.Context, log *zap.Logger) context.Context { + if ctx == nil { + return nil + } + return context.WithValue(ctx, ctxReqLoggerKeyType{}, log) +} + +// GetReqLog returns log if set. +// If zap.Logger isn't set returns nil. +func GetReqLog(ctx context.Context) *zap.Logger { + if ctx == nil { + return nil + } else if r, ok := ctx.Value(ctxReqLoggerKeyType{}).(*zap.Logger); ok { + return r + } + return nil +} + +// GetReqLogOrDefault returns log from context, if it exists. +// If the log is missing from the context, the default logger is returned. +func GetReqLogOrDefault(ctx context.Context, defaultLog *zap.Logger) *zap.Logger { + log := GetReqLog(ctx) + if log == nil { + log = defaultLog + } + return log +} From 70846fdaecf6ee2867507ac8e75b82ac61cc35db Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Mon, 21 Oct 2024 11:39:14 +0300 Subject: [PATCH 475/548] [#157] Support the continuous use of interceptors We can always add interceptors to the grpc connection to the storage, since the actual use will be controlled by the configuration from the frostfs-observability library. Signed-off-by: Roman Loginov --- cmd/http-gw/settings.go | 16 +++++----------- docs/gate-configuration.md | 2 +- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index eab5b6b..476d658 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -589,18 +589,12 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool. prmTree.SetMaxRequestAttempts(cfg.GetInt(cfgTreePoolMaxAttempts)) - var apiGRPCDialOpts []grpc.DialOption - var treeGRPCDialOpts []grpc.DialOption - if cfg.GetBool(cfgTracingEnabled) { - interceptors := []grpc.DialOption{ - grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), - grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), - } - treeGRPCDialOpts = append(treeGRPCDialOpts, interceptors...) - apiGRPCDialOpts = append(apiGRPCDialOpts, interceptors...) + interceptors := []grpc.DialOption{ + grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), + grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), } - prm.SetGRPCDialOptions(apiGRPCDialOpts...) - prmTree.SetGRPCDialOptions(treeGRPCDialOpts...) + prm.SetGRPCDialOptions(interceptors...) + prmTree.SetGRPCDialOptions(interceptors...) p, err := pool.NewPool(prm) if err != nil { diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index e8d1f4b..b484f9d 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -271,7 +271,7 @@ tracing: | Parameter | Type | SIGHUP reload | Default value | Description | |--------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| -| `enabled` | `bool` | no | `false` | Flag to enable the tracing. | +| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | | `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | | `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | | `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | From 8dc527296515bd255ab8d6920b21a1e2133eea98 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 24 Oct 2024 16:17:28 +0300 Subject: [PATCH 476/548] [#158] Rework app settings Update settings by sighup using one lock/unlock operation Signed-off-by: Denis Kirillov --- cmd/http-gw/app.go | 150 ++++++++++++---------------------------- cmd/http-gw/settings.go | 32 +++++++++ 2 files changed, 76 insertions(+), 106 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index b26df89..ead4a29 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -6,13 +6,11 @@ import ( "crypto/x509" "errors" "fmt" - "io" "net/http" "os" "os/signal" "runtime/debug" "strconv" - "strings" "sync" "syscall" "time" @@ -159,23 +157,47 @@ func newApp(ctx context.Context, opt ...Option) App { a.initResolver() a.initMetrics() a.initTracing(ctx) - a.loadIndexPageTemplate() return a } +func (a *app) initAppSettings() { + a.settings = &appSettings{ + reconnectInterval: fetchReconnectInterval(a.cfg), + } + a.settings.update(a.cfg, a.log) +} + +func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { + defaultTimestamp := v.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) + zipCompression := v.GetBool(cfgZipCompression) + returnIndexPage := v.GetBool(cfgIndexPageEnabled) + clientCut := v.GetBool(cfgClientCut) + bufferMaxSizeForPut := v.GetUint64(cfgBufferMaxSizeForPut) + namespaceHeader := v.GetString(cfgResolveNamespaceHeader) + defaultNamespaces := fetchDefaultNamespaces(v) + indexPage, indexEnabled := fetchIndexPageTemplate(v, l) + + s.mu.Lock() + defer s.mu.Unlock() + + s.defaultTimestamp = defaultTimestamp + s.zipCompression = zipCompression + s.returnIndexPage = returnIndexPage + s.clientCut = clientCut + s.bufferMaxSizeForPut = bufferMaxSizeForPut + s.namespaceHeader = namespaceHeader + s.defaultNamespaces = defaultNamespaces + s.returnIndexPage = indexEnabled + s.indexPageTemplate = indexPage +} + func (s *appSettings) DefaultTimestamp() bool { s.mu.RLock() defer s.mu.RUnlock() return s.defaultTimestamp } -func (s *appSettings) setDefaultTimestamp(val bool) { - s.mu.Lock() - s.defaultTimestamp = val - s.mu.Unlock() -} - func (s *appSettings) ZipCompression() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -197,73 +219,33 @@ func (s *appSettings) IndexPageTemplate() string { return s.indexPageTemplate } -func (s *appSettings) setZipCompression(val bool) { - s.mu.Lock() - s.zipCompression = val - s.mu.Unlock() -} - -func (s *appSettings) setReturnIndexPage(val bool) { - s.mu.Lock() - s.returnIndexPage = val - s.mu.Unlock() -} - -func (s *appSettings) setIndexTemplate(val string) { - s.mu.Lock() - s.indexPageTemplate = val - s.mu.Unlock() -} - -func (a *app) loadIndexPageTemplate() { - if !a.settings.IndexPageEnabled() { - return - } - reader, err := os.Open(a.cfg.GetString(cfgIndexPageTemplatePath)) - if err != nil { - a.settings.setIndexTemplate("") - a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) - return - } - tmpl, err := io.ReadAll(reader) - if err != nil { - a.settings.setIndexTemplate("") - a.log.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) - return - } - a.settings.setIndexTemplate(string(tmpl)) - a.log.Info(logs.SetCustomIndexPageTemplate) -} - func (s *appSettings) ClientCut() bool { s.mu.RLock() defer s.mu.RUnlock() return s.clientCut } -func (s *appSettings) setClientCut(val bool) { - s.mu.Lock() - s.clientCut = val - s.mu.Unlock() -} - func (s *appSettings) BufferMaxSizeForPut() uint64 { s.mu.RLock() defer s.mu.RUnlock() return s.bufferMaxSizeForPut } -func (s *appSettings) setBufferMaxSizeForPut(val uint64) { - s.mu.Lock() - s.bufferMaxSizeForPut = val - s.mu.Unlock() +func (s *appSettings) NamespaceHeader() string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.namespaceHeader } -func (a *app) initAppSettings() { - a.settings = &appSettings{ - reconnectInterval: fetchReconnectInterval(a.cfg), +func (s *appSettings) FormContainerZone(ns string) (zone string, isDefault bool) { + s.mu.RLock() + namespaces := s.defaultNamespaces + s.mu.RUnlock() + if slices.Contains(namespaces, ns) { + return v2container.SysAttributeZoneDefault, true } - a.updateSettings() + + return ns + ".ns", false } func (a *app) initResolver() { @@ -539,26 +521,15 @@ func (a *app) configReload(ctx context.Context) { a.stopServices() a.startServices() - a.updateSettings() + a.settings.update(a.cfg, a.log) a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled)) a.initTracing(ctx) - a.loadIndexPageTemplate() a.setHealthStatus() a.log.Info(logs.SIGHUPConfigReloadCompleted) } -func (a *app) updateSettings() { - a.settings.setDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) - a.settings.setZipCompression(a.cfg.GetBool(cfgZipCompression)) - a.settings.setReturnIndexPage(a.cfg.GetBool(cfgIndexPageEnabled)) - a.settings.setClientCut(a.cfg.GetBool(cfgClientCut)) - a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut)) - a.settings.setNamespaceHeader(a.cfg.GetString(cfgResolveNamespaceHeader)) - a.settings.setDefaultNamespaces(a.cfg.GetStringSlice(cfgResolveDefaultNamespaces)) -} - func (a *app) startServices() { pprofConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPprofEnabled), Address: a.cfg.GetString(cfgPprofAddress)} pprofService := metrics.NewPprofService(a.log, pprofConfig) @@ -847,39 +818,6 @@ func (a *app) setRuntimeParameters() { } } -func (s *appSettings) NamespaceHeader() string { - s.mu.RLock() - defer s.mu.RUnlock() - return s.namespaceHeader -} - -func (s *appSettings) setNamespaceHeader(nsHeader string) { - s.mu.Lock() - s.namespaceHeader = nsHeader - s.mu.Unlock() -} - -func (s *appSettings) FormContainerZone(ns string) (zone string, isDefault bool) { - s.mu.RLock() - namespaces := s.defaultNamespaces - s.mu.RUnlock() - if slices.Contains(namespaces, ns) { - return v2container.SysAttributeZoneDefault, true - } - - return ns + ".ns", false -} - -func (s *appSettings) setDefaultNamespaces(namespaces []string) { - for i := range namespaces { // to be set namespaces in env variable as `HTTP_GW_RESOLVE_BUCKET_DEFAULT_NAMESPACES="" "root"` - namespaces[i] = strings.Trim(namespaces[i], "\"") - } - - s.mu.Lock() - s.defaultNamespaces = namespaces - s.mu.Unlock() -} - func (a *app) scheduleReconnect(ctx context.Context, srv *fasthttp.Server) { go func() { t := time.NewTicker(a.settings.reconnectInterval) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 476d658..e777f67 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "io" "math" "os" "path" @@ -505,6 +506,37 @@ func fetchReconnectInterval(cfg *viper.Viper) time.Duration { return reconnect } +func fetchIndexPageTemplate(v *viper.Viper, l *zap.Logger) (string, bool) { + if !v.GetBool(cfgIndexPageEnabled) { + return "", false + } + + reader, err := os.Open(v.GetString(cfgIndexPageTemplatePath)) + if err != nil { + l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + return "", true + } + + tmpl, err := io.ReadAll(reader) + if err != nil { + l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + return "", true + } + + l.Info(logs.SetCustomIndexPageTemplate) + return string(tmpl), true +} + +func fetchDefaultNamespaces(v *viper.Viper) []string { + namespaces := v.GetStringSlice(cfgResolveDefaultNamespaces) + + for i := range namespaces { // to be set namespaces in env variable as `HTTP_GW_RESOLVE_BUCKET_DEFAULT_NAMESPACES="" "root"` + namespaces[i] = strings.Trim(namespaces[i], "\"") + } + + return namespaces +} + func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { var servers []ServerInfo seen := make(map[string]struct{}) From 901b8ff95b6ca8eb96880dc663b86014d8915d56 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 24 Oct 2024 16:18:24 +0300 Subject: [PATCH 477/548] [#158] Fix integration test compilation error Signed-off-by: Denis Kirillov --- cmd/http-gw/integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index e888ed6..79a2da5 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -102,7 +102,7 @@ func runServer(pathToWallet string) (App, context.CancelFunc) { v.Set(cfgWalletPath, pathToWallet) v.Set(cfgWalletPassphrase, "") - l, lvl := newStdoutLogger(zapcore.DebugLevel) + l, lvl := newStdoutLogger(v, zapcore.DebugLevel) application := newApp(cancelCtx, WithConfig(v), WithLogger(l, lvl)) go application.Serve() @@ -525,7 +525,7 @@ func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID id, err := clientPool.PutObject(ctx, prm) require.NoError(t, err) - return id + return id.ObjectID } func makeBearerToken(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string) string { From 46c63edd67af9f1676283db5d89e6b72f7bd32d4 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 24 Oct 2024 17:27:32 +0300 Subject: [PATCH 478/548] [#158] Support cors Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + cmd/http-gw/app.go | 158 +++++++++++++++++++++++++++++++++---- cmd/http-gw/settings.go | 19 +++++ config/config.env | 7 ++ config/config.yaml | 8 ++ docs/gate-configuration.md | 24 ++++++ 6 files changed, 201 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46e9c23..0990b51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This document outlines major changes between releases. ### Added - Support percent-encoding for GET queries (#134) - Add `trace_id` to logs (#148) +- Add `cors` config params (#158) ### Changed - Update go version to 1.22 (#132) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index ead4a29..fa93a0e 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -11,6 +11,7 @@ import ( "os/signal" "runtime/debug" "strconv" + "strings" "sync" "syscall" "time" @@ -87,15 +88,30 @@ type ( appSettings struct { reconnectInterval time.Duration - mu sync.RWMutex - defaultTimestamp bool - zipCompression bool - clientCut bool - returnIndexPage bool - indexPageTemplate string - bufferMaxSizeForPut uint64 - namespaceHeader string - defaultNamespaces []string + mu sync.RWMutex + defaultTimestamp bool + zipCompression bool + clientCut bool + returnIndexPage bool + indexPageTemplate string + bufferMaxSizeForPut uint64 + namespaceHeader string + defaultNamespaces []string + corsAllowOrigin string + corsAllowMethods []string + corsAllowHeaders []string + corsExposeHeaders []string + corsAllowCredentials bool + corsMaxAge int + } + + CORS struct { + AllowOrigin string + AllowMethods []string + AllowHeaders []string + ExposeHeaders []string + AllowCredentials bool + MaxAge int } ) @@ -177,6 +193,12 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { namespaceHeader := v.GetString(cfgResolveNamespaceHeader) defaultNamespaces := fetchDefaultNamespaces(v) indexPage, indexEnabled := fetchIndexPageTemplate(v, l) + corsAllowOrigin := v.GetString(cfgCORSAllowOrigin) + corsAllowMethods := v.GetStringSlice(cfgCORSAllowMethods) + corsAllowHeaders := v.GetStringSlice(cfgCORSAllowHeaders) + corsExposeHeaders := v.GetStringSlice(cfgCORSExposeHeaders) + corsAllowCredentials := v.GetBool(cfgCORSAllowCredentials) + corsMaxAge := fetchCORSMaxAge(v) s.mu.Lock() defer s.mu.Unlock() @@ -190,6 +212,12 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { s.defaultNamespaces = defaultNamespaces s.returnIndexPage = indexEnabled s.indexPageTemplate = indexPage + s.corsAllowOrigin = corsAllowOrigin + s.corsAllowMethods = corsAllowMethods + s.corsAllowHeaders = corsAllowHeaders + s.corsExposeHeaders = corsExposeHeaders + s.corsAllowCredentials = corsAllowCredentials + s.corsMaxAge = corsMaxAge } func (s *appSettings) DefaultTimestamp() bool { @@ -219,6 +247,29 @@ func (s *appSettings) IndexPageTemplate() string { return s.indexPageTemplate } +func (s *appSettings) CORS() CORS { + s.mu.RLock() + defer s.mu.RUnlock() + + allowMethods := make([]string, len(s.corsAllowMethods)) + copy(allowMethods, s.corsAllowMethods) + + allowHeaders := make([]string, len(s.corsAllowHeaders)) + copy(allowHeaders, s.corsAllowHeaders) + + exposeHeaders := make([]string, len(s.corsExposeHeaders)) + copy(exposeHeaders, s.corsExposeHeaders) + + return CORS{ + AllowOrigin: s.corsAllowOrigin, + AllowMethods: allowMethods, + AllowHeaders: allowHeaders, + ExposeHeaders: exposeHeaders, + AllowCredentials: s.corsAllowCredentials, + MaxAge: s.corsMaxAge, + } +} + func (s *appSettings) ClientCut() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -550,7 +601,6 @@ func (a *app) stopServices() { svc.ShutDown(ctx) } } - func (a *app) configureRouter(handler *handler.Handler) { r := router.New() r.RedirectTrailingSlash = true @@ -561,20 +611,96 @@ func (a *app) configureRouter(handler *handler.Handler) { response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.Upload)))))) + r.POST("/upload/{cid}", a.addMiddlewares(handler.Upload)) + r.OPTIONS("/upload/{cid}", a.addPreflight()) a.log.Info(logs.AddedPathUploadCid) - r.GET("/get/{cid}/{oid:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadByAddressOrBucketName)))))) - r.HEAD("/get/{cid}/{oid:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.HeadByAddressOrBucketName)))))) + r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(handler.DownloadByAddressOrBucketName)) + r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(handler.HeadByAddressOrBucketName)) + r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetCidOid) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadByAttribute)))))) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.HeadByAttribute)))))) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(handler.DownloadByAttribute)) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(handler.HeadByAttribute)) + r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.tracer(a.logger(a.canonicalizer(a.tokenizer(a.reqNamespace(handler.DownloadZipped)))))) + r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(handler.DownloadZipped)) + r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) a.log.Info(logs.AddedPathZipCidPrefix) a.webServer.Handler = r.Handler } +func (a *app) addMiddlewares(h fasthttp.RequestHandler) fasthttp.RequestHandler { + list := []func(fasthttp.RequestHandler) fasthttp.RequestHandler{ + a.tracer, + a.logger, + a.canonicalizer, + a.tokenizer, + a.reqNamespace, + a.cors, + } + + for i := len(list) - 1; i >= 0; i-- { + h = list[i](h) + } + + return h +} + +func (a *app) addPreflight() fasthttp.RequestHandler { + list := []func(fasthttp.RequestHandler) fasthttp.RequestHandler{ + a.tracer, + a.logger, + a.reqNamespace, + } + + h := a.preflightHandler + for i := len(list) - 1; i >= 0; i-- { + h = list[i](h) + } + + return h +} + +func (a *app) preflightHandler(c *fasthttp.RequestCtx) { + cors := a.settings.CORS() + setCORSHeaders(c, cors) +} + +func (a *app) cors(h fasthttp.RequestHandler) fasthttp.RequestHandler { + return func(c *fasthttp.RequestCtx) { + h(c) + code := c.Response.StatusCode() + if code >= fasthttp.StatusOK && code < fasthttp.StatusMultipleChoices { + cors := a.settings.CORS() + setCORSHeaders(c, cors) + } + } +} + +func setCORSHeaders(c *fasthttp.RequestCtx, cors CORS) { + c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(cors.MaxAge)) + + if len(cors.AllowOrigin) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, cors.AllowOrigin) + } + + if len(cors.AllowMethods) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(cors.AllowMethods, ",")) + } + + if len(cors.AllowHeaders) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, strings.Join(cors.AllowHeaders, ",")) + } + + if len(cors.ExposeHeaders) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(cors.ExposeHeaders, ",")) + } + + if cors.AllowCredentials { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + } +} + func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { requiredFields := []zap.Field{zap.Uint64("id", req.ID())} diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index e777f67..5fb628d 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -56,6 +56,8 @@ const ( defaultReconnectInterval = time.Minute + defaultCORSMaxAge = 600 // seconds + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" @@ -141,6 +143,14 @@ const ( cfgResolveNamespaceHeader = "resolve_bucket.namespace_header" cfgResolveDefaultNamespaces = "resolve_bucket.default_namespaces" + // CORS. + cfgCORSAllowOrigin = "cors.allow_origin" + cfgCORSAllowMethods = "cors.allow_methods" + cfgCORSAllowHeaders = "cors.allow_headers" + cfgCORSExposeHeaders = "cors.expose_headers" + cfgCORSAllowCredentials = "cors.allow_credentials" + cfgCORSMaxAge = "cors.max_age" + // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -537,6 +547,15 @@ func fetchDefaultNamespaces(v *viper.Viper) []string { return namespaces } +func fetchCORSMaxAge(v *viper.Viper) int { + maxAge := v.GetInt(cfgCORSMaxAge) + if maxAge <= 0 { + maxAge = defaultCORSMaxAge + } + + return maxAge +} + func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { var servers []ServerInfo seen := make(map[string]struct{}) diff --git a/config/config.env b/config/config.env index e1d5e7d..d2f4a56 100644 --- a/config/config.env +++ b/config/config.env @@ -126,3 +126,10 @@ HTTP_GW_RESOLVE_BUCKET_DEFAULT_NAMESPACES="" "root" # Max attempt to make successful tree request. # default value is 0 that means the number of attempts equals to number of nodes in pool. HTTP_GW_FROSTFS_TREE_POOL_MAX_ATTEMPTS=0 + +HTTP_GW_CORS_ALLOW_ORIGIN="*" +HTTP_GW_CORS_ALLOW_METHODS="GET" "POST" +HTTP_GW_CORS_ALLOW_HEADERS="*" +HTTP_GW_CORS_EXPOSE_HEADERS="*" +HTTP_GW_CORS_ALLOW_CREDENTIALS=false +HTTP_GW_CORS_MAX_AGE=600 diff --git a/config/config.yaml b/config/config.yaml index 61aa70b..dd985ad 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -138,3 +138,11 @@ cache: resolve_bucket: namespace_header: X-Frostfs-Namespace default_namespaces: [ "", "root" ] + +cors: + allow_origin: "" + allow_methods: [] + allow_headers: [] + expose_headers: [] + allow_credentials: false + max_age: 600 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index b484f9d..7a3eba7 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -363,3 +363,27 @@ index_page: |-----------------|----------|---------------|---------------|---------------------------------------------------------------------------------| | `enabled` | `bool` | yes | `false` | Flag to enable index_page return if no object with specified S3-name was found. | | `template_path` | `string` | yes | `""` | Path to .gotmpl file with html template for index_page. | + +# `cors` section + +Parameters for CORS (used in OPTIONS requests and responses in all handlers). +If values are not set, headers will not be included to response. + +```yaml +cors: + allow_origin: "*" + allow_methods: ["GET", "HEAD"] + allow_headers: ["Authorization"] + expose_headers: ["*"] + allow_credentials: false + max_age: 600 +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------------|------------|---------------|---------------|--------------------------------------------------------| +| `allow_origin` | `string` | yes | | Values for `Access-Control-Allow-Origin` headers. | +| `allow_methods` | `[]string` | yes | | Values for `Access-Control-Allow-Methods` headers. | +| `allow_headers` | `[]string` | yes | | Values for `Access-Control-Allow-Headers` headers. | +| `expose_headers` | `[]string` | yes | | Values for `Access-Control-Expose-Headers` headers. | +| `allow_credentials` | `bool` | yes | `false` | Values for `Access-Control-Allow-Credentials` headers. | +| `max_age` | `int` | yes | `600` | Values for `Access-Control-Max-Age ` headers. | From 69b7761bd6e03cad3f70bffe592716ac8bb52489 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 29 Oct 2024 17:45:42 +0300 Subject: [PATCH 479/548] [#160] Add internal/net package with multinet dialer source Signed-off-by: Alex Vanin --- go.mod | 3 +- go.sum | 2 + internal/logs/logs.go | 2 + internal/net/config.go | 68 ++++++++++++++++++++++++++++++++++ internal/net/dial_target.go | 54 +++++++++++++++++++++++++++ internal/net/dialer.go | 36 ++++++++++++++++++ internal/net/dialer_source.go | 69 +++++++++++++++++++++++++++++++++++ internal/net/event_handler.go | 28 ++++++++++++++ 8 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 internal/net/config.go create mode 100644 internal/net/dial_target.go create mode 100644 internal/net/dialer.go create mode 100644 internal/net/dialer_source.go create mode 100644 internal/net/event_handler.go diff --git a/go.mod b/go.mod index d1a3788..efb79eb 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 + git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 github.com/docker/go-units v0.4.0 @@ -25,6 +26,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/net v0.26.0 + golang.org/x/sys v0.22.0 google.golang.org/grpc v1.66.2 ) @@ -106,7 +108,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/go.sum b/go.sum index bc433eb..ce02ee6 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= +git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= +git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972/go.mod h1:2hM42MBrlhvN6XToaW6OWNk5ZLcu1FhaukGgxtfpDDI= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 96bdaa5..7b7ddc1 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -79,4 +79,6 @@ const ( ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" WarnDuplicateAddress = "duplicate address" + MultinetDialSuccess = "multinet dial successful" + MultinetDialFail = "multinet dial failed" ) diff --git a/internal/net/config.go b/internal/net/config.go new file mode 100644 index 0000000..b40e003 --- /dev/null +++ b/internal/net/config.go @@ -0,0 +1,68 @@ +package net + +import ( + "errors" + "fmt" + "net/netip" + "slices" + "time" + + "git.frostfs.info/TrueCloudLab/multinet" +) + +var errEmptySourceIPList = errors.New("empty source IP list") + +type Subnet struct { + Prefix string + SourceIPs []string +} + +type Config struct { + Enabled bool + Subnets []Subnet + Balancer string + Restrict bool + FallbackDelay time.Duration + EventHandler multinet.EventHandler +} + +func (c Config) toMultinetConfig() (multinet.Config, error) { + var subnets []multinet.Subnet + for _, s := range c.Subnets { + var ms multinet.Subnet + p, err := netip.ParsePrefix(s.Prefix) + if err != nil { + return multinet.Config{}, fmt.Errorf("parse IP prefix '%s': %w", s.Prefix, err) + } + ms.Prefix = p + for _, ip := range s.SourceIPs { + addr, err := netip.ParseAddr(ip) + if err != nil { + return multinet.Config{}, fmt.Errorf("parse IP address '%s': %w", ip, err) + } + ms.SourceIPs = append(ms.SourceIPs, addr) + } + if len(ms.SourceIPs) == 0 { + return multinet.Config{}, errEmptySourceIPList + } + subnets = append(subnets, ms) + } + return multinet.Config{ + Subnets: subnets, + Balancer: multinet.BalancerType(c.Balancer), + Restrict: c.Restrict, + FallbackDelay: c.FallbackDelay, + Dialer: newDefaultDialer(), + EventHandler: c.EventHandler, + }, nil +} + +func (c Config) equals(other Config) bool { + return c.Enabled == other.Enabled && + slices.EqualFunc(c.Subnets, other.Subnets, func(lhs, rhs Subnet) bool { + return lhs.Prefix == rhs.Prefix && slices.Equal(lhs.SourceIPs, rhs.SourceIPs) + }) && + c.Balancer == other.Balancer && + c.Restrict == other.Restrict && + c.FallbackDelay == other.FallbackDelay +} diff --git a/internal/net/dial_target.go b/internal/net/dial_target.go new file mode 100644 index 0000000..6265f18 --- /dev/null +++ b/internal/net/dial_target.go @@ -0,0 +1,54 @@ +// NOTE: code is taken from https://github.com/grpc/grpc-go/blob/v1.68.x/internal/transport/http_util.go + +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package net + +import ( + "net/url" + "strings" +) + +// parseDialTarget returns the network and address to pass to dialer. +func parseDialTarget(target string) (string, string) { + net := "tcp" + m1 := strings.Index(target, ":") + m2 := strings.Index(target, ":/") + // handle unix:addr which will fail with url.Parse + if m1 >= 0 && m2 < 0 { + if n := target[0:m1]; n == "unix" { + return n, target[m1+1:] + } + } + if m2 >= 0 { + t, err := url.Parse(target) + if err != nil { + return net, target + } + scheme := t.Scheme + addr := t.Path + if scheme == "unix" { + if addr == "" { + addr = t.Host + } + return scheme, addr + } + } + return net, target +} diff --git a/internal/net/dialer.go b/internal/net/dialer.go new file mode 100644 index 0000000..8441dd5 --- /dev/null +++ b/internal/net/dialer.go @@ -0,0 +1,36 @@ +package net + +import ( + "net" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +func newDefaultDialer() net.Dialer { + // From `grpc.WithContextDialer` comment: + // + // Note: All supported releases of Go (as of December 2023) override the OS + // defaults for TCP keepalive time and interval to 15s. To enable TCP keepalive + // with OS defaults for keepalive time and interval, use a net.Dialer that sets + // the KeepAlive field to a negative value, and sets the SO_KEEPALIVE socket + // option to true from the Control field. For a concrete example of how to do + // this, see internal.NetDialerWithTCPKeepalive(). + // + // https://github.com/grpc/grpc-go/blob/830135e6c5a351abf75f0c9cfdf978e5df8daeba/dialoptions.go#L432 + // + // From `internal.NetDialerWithTCPKeepalive` comment: + // + // TODO: Once https://github.com/golang/go/issues/62254 lands, and the + // appropriate Go version becomes less than our least supported Go version, we + // should look into using the new API to make things more straightforward. + return net.Dialer{ + KeepAlive: time.Duration(-1), + Control: func(_, _ string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + _ = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_KEEPALIVE, 1) + }) + }, + } +} diff --git a/internal/net/dialer_source.go b/internal/net/dialer_source.go new file mode 100644 index 0000000..e6a142a --- /dev/null +++ b/internal/net/dialer_source.go @@ -0,0 +1,69 @@ +package net + +import ( + "context" + "net" + "sync" + + "git.frostfs.info/TrueCloudLab/multinet" +) + +type DialerSource struct { + guard sync.RWMutex + + c Config + + md multinet.Dialer +} + +func NewDialerSource(c Config) (*DialerSource, error) { + result := &DialerSource{} + if err := result.build(c); err != nil { + return nil, err + } + return result, nil +} + +func (s *DialerSource) build(c Config) error { + if c.Enabled { + mc, err := c.toMultinetConfig() + if err != nil { + return err + } + md, err := multinet.NewDialer(mc) + if err != nil { + return err + } + s.md = md + s.c = c + return nil + } + s.md = nil + s.c = c + return nil +} + +// GrpcContextDialer returns grpc.WithContextDialer func. +// Returns nil if multinet disabled. +func (s *DialerSource) GrpcContextDialer() func(context.Context, string) (net.Conn, error) { + s.guard.RLock() + defer s.guard.RUnlock() + + if s.c.Enabled { + return func(ctx context.Context, address string) (net.Conn, error) { + network, address := parseDialTarget(address) + return s.md.DialContext(ctx, network, address) + } + } + return nil +} + +func (s *DialerSource) Update(c Config) error { + s.guard.Lock() + defer s.guard.Unlock() + + if s.c.equals(c) { + return nil + } + return s.build(c) +} diff --git a/internal/net/event_handler.go b/internal/net/event_handler.go new file mode 100644 index 0000000..9520c01 --- /dev/null +++ b/internal/net/event_handler.go @@ -0,0 +1,28 @@ +package net + +import ( + "net" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "go.uber.org/zap" +) + +type LogEventHandler struct { + logger *zap.Logger +} + +func (l LogEventHandler) DialPerformed(sourceIP net.Addr, _, address string, err error) { + sourceIPString := "undefined" + if sourceIP != nil { + sourceIPString = sourceIP.Network() + "://" + sourceIP.String() + } + if err == nil { + l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString), zap.String("destination", address)) + } else { + l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString), zap.String("destination", address), zap.Error(err)) + } +} + +func NewLogEventHandler(logger *zap.Logger) LogEventHandler { + return LogEventHandler{logger: logger} +} From 8bc64ce5e997b169b9c146b3ee291b86c3fdab62 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 29 Oct 2024 14:41:49 +0300 Subject: [PATCH 480/548] [#160] Use source dialer for gRPC connection to storage Signed-off-by: Alex Vanin --- cmd/http-gw/app.go | 12 +++++++++-- cmd/http-gw/settings.go | 47 ++++++++++++++++++++++++++++++++++++++++- internal/logs/logs.go | 2 ++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index fa93a0e..84379d4 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -21,6 +21,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" @@ -87,6 +88,7 @@ type ( // appSettings stores reloading parameters, so it has to provide getters and setters which use RWMutex. appSettings struct { reconnectInterval time.Duration + dialerSource *internalnet.DialerSource mu sync.RWMutex defaultTimestamp bool @@ -148,6 +150,8 @@ func newApp(ctx context.Context, opt ...Option) App { opt[i](a) } + a.initAppSettings() + // -- setup FastHTTP server -- a.webServer.Name = "frost-http-gw" a.webServer.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) @@ -161,7 +165,7 @@ func newApp(ctx context.Context, opt ...Option) App { a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - a.pool, a.treePool, a.key = getPools(ctx, a.log, a.cfg) + a.pool, a.treePool, a.key = getPools(ctx, a.log, a.cfg, a.settings.dialerSource) var owner user.ID user.IDFromKey(&owner, a.key.PrivateKey.PublicKey) @@ -169,7 +173,6 @@ func newApp(ctx context.Context, opt ...Option) App { a.setRuntimeParameters() - a.initAppSettings() a.initResolver() a.initMetrics() a.initTracing(ctx) @@ -180,6 +183,7 @@ func newApp(ctx context.Context, opt ...Option) App { func (a *app) initAppSettings() { a.settings = &appSettings{ reconnectInterval: fetchReconnectInterval(a.cfg), + dialerSource: getDialerSource(a.log, a.cfg), } a.settings.update(a.cfg, a.log) } @@ -559,6 +563,10 @@ func (a *app) configReload(ctx context.Context) { a.logLevel.SetLevel(lvl) } + if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.cfg, a.log)); err != nil { + a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err)) + } + if err := a.resolver.UpdateResolvers(a.getResolverConfig()); err != nil { a.log.Warn(logs.FailedToUpdateResolvers, zap.Error(err)) } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 5fb628d..f464fbc 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -16,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" @@ -58,6 +59,8 @@ const ( defaultCORSMaxAge = 600 // seconds + defaultMultinetFallbackDelay = 300 * time.Millisecond + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" @@ -151,6 +154,13 @@ const ( cfgCORSAllowCredentials = "cors.allow_credentials" cfgCORSMaxAge = "cors.max_age" + // Multinet. + cfgMultinetEnabled = "multinet.enabled" + cfgMultinetBalancer = "multinet.balancer" + cfgMultinetRestrict = "multinet.restrict" + cfgMultinetFallbackDelay = "multinet.fallback_delay" + cfgMultinetSubnets = "multinet.subnets" + // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -245,6 +255,9 @@ func settings() *viper.Viper { v.SetDefault(cfgResolveNamespaceHeader, defaultNamespaceHeader) v.SetDefault(cfgResolveDefaultNamespaces, []string{"", "root"}) + // multinet + v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay) + // Binding flags if err := v.BindPFlag(cfgPprofEnabled, flags.Lookup(cmdPprof)); err != nil { panic(err) @@ -584,7 +597,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { return servers } -func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) { +func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper, dialSource *internalnet.DialerSource) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) { key, err := getFrostFSKey(cfg, logger) if err != nil { logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) @@ -643,6 +656,7 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool. interceptors := []grpc.DialOption{ grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), + grpc.WithContextDialer(dialSource.GrpcContextDialer()), } prm.SetGRPCDialOptions(interceptors...) prmTree.SetGRPCDialOptions(interceptors...) @@ -745,3 +759,34 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue return defaultValue } + +func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSource { + source, err := internalnet.NewDialerSource(fetchMultinetConfig(cfg, logger)) + if err != nil { + logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err)) + } + return source +} + +func fetchMultinetConfig(v *viper.Viper, l *zap.Logger) (cfg internalnet.Config) { + cfg.Enabled = v.GetBool(cfgMultinetEnabled) + cfg.Balancer = v.GetString(cfgMultinetBalancer) + cfg.Restrict = v.GetBool(cfgMultinetRestrict) + cfg.FallbackDelay = v.GetDuration(cfgMultinetFallbackDelay) + cfg.Subnets = make([]internalnet.Subnet, 0, 5) + cfg.EventHandler = internalnet.NewLogEventHandler(l) + + for i := 0; ; i++ { + key := cfgMultinetSubnets + "." + strconv.Itoa(i) + "." + subnet := internalnet.Subnet{} + + subnet.Prefix = v.GetString(key + "mask") + if subnet.Prefix == "" { + break + } + subnet.SourceIPs = v.GetStringSlice(key + "source_ips") + cfg.Subnets = append(cfg.Subnets, subnet) + } + + return +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 7b7ddc1..409f87d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -81,4 +81,6 @@ const ( WarnDuplicateAddress = "duplicate address" MultinetDialSuccess = "multinet dial successful" MultinetDialFail = "multinet dial failed" + FailedToLoadMultinetConfig = "failed to load multinet config" + MultinetConfigWontBeUpdated = "multinet config won't be updated" ) From 821f8c2248040415cb3984e39afd38ff9acebdd5 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 29 Oct 2024 14:44:09 +0300 Subject: [PATCH 481/548] [#160] Add documentation for multinet settings Signed-off-by: Alex Vanin --- config/config.env | 13 +++++++++++++ config/config.yaml | 17 ++++++++++++++++ docs/gate-configuration.md | 40 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/config/config.env b/config/config.env index d2f4a56..4fd8132 100644 --- a/config/config.env +++ b/config/config.env @@ -133,3 +133,16 @@ HTTP_GW_CORS_ALLOW_HEADERS="*" HTTP_GW_CORS_EXPOSE_HEADERS="*" HTTP_GW_CORS_ALLOW_CREDENTIALS=false HTTP_GW_CORS_MAX_AGE=600 + +# Multinet properties +# Enable multinet support +HTTP_GW_MULTINET_ENABLED=false +# Strategy to pick source IP address +HTTP_GW_MULTINET_BALANCER=roundrobin +# Restrict requests with unknown destination subnet +HTTP_GW_MULTINET_RESTRICT=false +# Delay between ipv6 to ipv4 fallback switch +HTTP_GW_MULTINET_FALLBACK_DELAY=300ms +# List of subnets and IP addresses to use as source for those subnets +HTTP_GW_MULTINET_SUBNETS_1_MASK=1.2.3.4/24 +HTTP_GW_MULTINET_SUBNETS_1_SOURCE_IPS=1.2.3.4 1.2.3.5 diff --git a/config/config.yaml b/config/config.yaml index dd985ad..9169acc 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -146,3 +146,20 @@ cors: expose_headers: [] allow_credentials: false max_age: 600 + +# Multinet properties +multinet: + # Enable multinet support + enabled: false + # Strategy to pick source IP address + balancer: roundrobin + # Restrict requests with unknown destination subnet + restrict: false + # Delay between ipv6 to ipv4 fallback switch + fallback_delay: 300ms + # List of subnets and IP addresses to use as source for those subnets + subnets: + - mask: 1.2.3.4/24 + source_ips: + - 1.2.3.4 + - 1.2.3.5 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 7a3eba7..be4b30b 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -58,6 +58,7 @@ $ cat http.log | `cache` | [Cache configuration](#cache-section) | | `resolve_bucket` | [Bucket name resolving configuration](#resolve_bucket-section) | | `index_page` | [Index page configuration](#index_page-section) | +| `multinet` | [Multinet configuration](#multinet-section) | # General section @@ -387,3 +388,42 @@ cors: | `expose_headers` | `[]string` | yes | | Values for `Access-Control-Expose-Headers` headers. | | `allow_credentials` | `bool` | yes | `false` | Values for `Access-Control-Allow-Credentials` headers. | | `max_age` | `int` | yes | `600` | Values for `Access-Control-Max-Age ` headers. | + +# `multinet` section + +Configuration of multinet support. + +```yaml +multinet: + enabled: false + balancer: roundrobin + restrict: false + fallback_delay: 300ms + subnets: + - mask: 1.2.3.4/24 + source_ips: + - 1.2.3.4 + - 1.2.3.5 +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|------------------|--------------------------------|---------------|---------------|--------------------------------------------------------------------------------------------| +| `enabled` | `bool` | yes | `false` | Enables multinet setting to manage source ip of outcoming requests. | +| `balancer` | `string` | yes | `""` | Strategy to pick source IP. By default picks first address. Supports `roundrobin` setting. | +| `restrict` | `bool` | yes | `false` | Restricts requests to an undefined subnets. | +| `fallback_delay` | `duration` | yes | `300ms` | Delay between IPv6 and IPv4 fallback stack switch. | +| `subnets` | [[]Subnet](#subnet-subsection) | yes | | Set of subnets to apply multinet dial settings. | + +#### `subnet` subsection + +```yaml +- mask: 1.2.3.4/24 + source_ips: + - 1.2.3.4 + - 1.2.3.5 +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|--------------|------------|---------------|---------------|----------------------------------------------------------------------| +| `mask` | `string` | yes | | Destination subnet. | +| `source_ips` | `[]string` | yes | | Array of source IP addresses to use when dialing destination subnet. | From 679731ee52f72cf060b731d1ccd24e357b121f2c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 5 Nov 2024 17:50:50 +0300 Subject: [PATCH 482/548] [#161] Update SDK Need fix https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pulls/282 Signed-off-by: Denis Kirillov --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index efb79eb..9048009 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.22 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 @@ -24,7 +24,7 @@ require ( go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/trace v1.28.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/net v0.26.0 golang.org/x/sys v0.22.0 google.golang.org/grpc v1.66.2 @@ -41,7 +41,7 @@ require ( github.com/Microsoft/hcsshim v0.9.2 // indirect github.com/VictoriaMetrics/easyproto v0.1.4 // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index ce02ee6..b6069d2 100644 --- a/go.sum +++ b/go.sum @@ -37,16 +37,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e h1:740ABnOBYx4o6jxULHdSSnVW2fYIO35ohg+Uz59sxd0= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 h1:ivcdxQeQDnx4srF2ezoaeVlF0FAycSAztwfIUJnUI4s= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 h1:ijUci3thz0EwWkuRJDocW5D1RkVAJlt9xNG4CYepC90= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 h1:f7jan6eBDN88DKnKj8GKyWpfjBbSzjDALcDejYKRgCs= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3/go.mod h1:3txOjFJ8M/JFs01h7xOrnQHVn6hZgDNA16ivyUlu1iU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= @@ -114,8 +114,8 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -928,8 +928,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From d5b92446bd34ef518f5687b2c607efadbf1ff8d6 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Wed, 6 Nov 2024 11:38:03 +0300 Subject: [PATCH 483/548] [#162] Stop using obsolete .github directory This commit is a part of multi-repo cleanup effort: https://git.frostfs.info/TrueCloudLab/frostfs-infra/issues/136 Signed-off-by: Vitaliy Potyarkin --- {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md | 0 {.github => .forgejo}/ISSUE_TEMPLATE/config.yml | 0 {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md | 0 {.github => .forgejo}/logo.svg | 0 .github/CODEOWNERS | 1 - CODEOWNERS | 1 + README.md | 2 +- 7 files changed, 2 insertions(+), 2 deletions(-) rename {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/config.yml (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md (100%) rename {.github => .forgejo}/logo.svg (100%) delete mode 100644 .github/CODEOWNERS create mode 100644 CODEOWNERS diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.forgejo/ISSUE_TEMPLATE/bug_report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.md rename to .forgejo/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.forgejo/ISSUE_TEMPLATE/config.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/config.yml rename to .forgejo/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.forgejo/ISSUE_TEMPLATE/feature_request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .forgejo/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/logo.svg b/.forgejo/logo.svg similarity index 100% rename from .github/logo.svg rename to .forgejo/logo.svg diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index c280648..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @alexvanin @dkirillov diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..43df11e --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +.* @alexvanin @dkirillov diff --git a/README.md b/README.md index 019b8ff..e1af0eb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

-FrostFS logo +FrostFS logo

FrostFS is a decentralized distributed object storage integrated with the NEO Blockchain. From 22d905e51ec963c388aba8930e66d4c4a866aacf Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Fri, 15 Nov 2024 14:30:23 +0300 Subject: [PATCH 484/548] [#165] Execute CI on push to master Discussion: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/issues/550 Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/builds.yml | 6 +++++- .forgejo/workflows/tests.yml | 6 +++++- .forgejo/workflows/vulncheck.yml | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.forgejo/workflows/builds.yml b/.forgejo/workflows/builds.yml index 490a97c..7c2bb04 100644 --- a/.forgejo/workflows/builds.yml +++ b/.forgejo/workflows/builds.yml @@ -1,4 +1,8 @@ -on: [pull_request] +on: + pull_request: + push: + branches: + - master jobs: builds: diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index db7f986..81d93dc 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -1,4 +1,8 @@ -on: [pull_request] +on: + pull_request: + push: + branches: + - master jobs: lint: diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 7a82bc3..76e2965 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -1,4 +1,8 @@ -on: [pull_request] +on: + pull_request: + push: + branches: + - master jobs: vulncheck: From 9c0b499ea6b8ea7ef8bdbadd254415b46b579b9a Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Fri, 15 Nov 2024 10:53:46 +0300 Subject: [PATCH 485/548] [#164] Add tracing attributes Signed-off-by: Roman Loginov --- cmd/http-gw/app.go | 7 +++++++ cmd/http-gw/settings.go | 33 +++++++++++++++++++++++++++++---- config/config.env | 4 ++++ config/config.yaml | 6 ++++++ docs/gate-configuration.md | 34 ++++++++++++++++++++++++++++------ go.mod | 2 +- go.sum | 4 ++-- 7 files changed, 77 insertions(+), 13 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 84379d4..853977c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -927,6 +927,13 @@ func (a *app) initTracing(ctx context.Context) { cfg.ServerCaCertPool = certPool } + attributes, err := fetchTracingAttributes(a.cfg) + if err != nil { + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + return + } + cfg.Attributes = attributes + updated, err := tracing.Setup(ctx, cfg) if err != nil { a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index f464fbc..4f1712b 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -86,10 +86,11 @@ const ( cfgPprofAddress = "pprof.address" // Tracing ... - cfgTracingEnabled = "tracing.enabled" - cfgTracingExporter = "tracing.exporter" - cfgTracingEndpoint = "tracing.endpoint" - cfgTracingTrustedCa = "tracing.trusted_ca" + cfgTracingEnabled = "tracing.enabled" + cfgTracingExporter = "tracing.exporter" + cfgTracingEndpoint = "tracing.endpoint" + cfgTracingTrustedCa = "tracing.trusted_ca" + cfgTracingAttributes = "tracing.attributes" // Pool config. cfgConTimeout = "connect_timeout" @@ -790,3 +791,27 @@ func fetchMultinetConfig(v *viper.Viper, l *zap.Logger) (cfg internalnet.Config) return } + +func fetchTracingAttributes(v *viper.Viper) (map[string]string, error) { + attributes := make(map[string]string) + for i := 0; ; i++ { + key := cfgTracingAttributes + "." + strconv.Itoa(i) + "." + attrKey := v.GetString(key + "key") + attrValue := v.GetString(key + "value") + if attrKey == "" { + break + } + + if _, ok := attributes[attrKey]; ok { + return nil, fmt.Errorf("tracing attribute key %s defined more than once", attrKey) + } + + if attrValue == "" { + return nil, fmt.Errorf("empty tracing attribute value for key %s", attrKey) + } + + attributes[attrKey] = attrValue + } + + return attributes, nil +} diff --git a/config/config.env b/config/config.env index 4fd8132..db54c92 100644 --- a/config/config.env +++ b/config/config.env @@ -104,6 +104,10 @@ HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENDPOINT="localhost:4317" HTTP_GW_TRACING_EXPORTER="otlp_grpc" HTTP_GW_TRACING_TRUSTED_CA="" +HTTP_GW_TRACING_ATTRIBUTES_0_KEY=key0 +HTTP_GW_TRACING_ATTRIBUTES_0_VALUE=value +HTTP_GW_TRACING_ATTRIBUTES_1_KEY=key1 +HTTP_GW_TRACING_ATTRIBUTES_1_VALUE=value HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 diff --git a/config/config.yaml b/config/config.yaml index 9169acc..6c89e78 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -9,11 +9,17 @@ pprof: prometheus: enabled: false # Enable metrics. address: localhost:8084 + tracing: enabled: true exporter: "otlp_grpc" endpoint: "localhost:4317" trusted_ca: "" + attributes: + - key: key0 + value: value + - key: key1 + value: value logger: level: debug # Log level. diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index be4b30b..5b5b018 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -268,14 +268,36 @@ tracing: exporter: "otlp_grpc" endpoint: "localhost:4317" trusted_ca: "/etc/ssl/telemetry-trusted-ca.pem" + attributes: + - key: key0 + value: value + - key: key1 + value: value ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|--------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| -| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | -| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | -| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | -| `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | +| Parameter | Type | SIGHUP reload | Default value | Description | +| ------------ | -------------------------------------- | ------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | +| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | +| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | +| `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | +| `attributes` | [[]Attributes](#attributes-subsection) | yes | | An array of configurable attributes in key-value format. | + + +#### `attributes` subsection + +```yaml + attributes: + - key: key0 + value: value + - key: key1 + value: value +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------------------|----------|---------------|---------------|----------------------------------------------------------| +| `key` | `string` | yes | | Attribute key. | +| `value` | `string` | yes | | Attribute value. | # `runtime` section Contains runtime parameters. diff --git a/go.mod b/go.mod index 9048009..ca8f4fe 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 diff --git a/go.sum b/go.sum index b6069d2..197f21c 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 h1:f7jan6eBDN88DKnKj8GKyWpfjBbSzjDALcDejYKRgCs= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3/go.mod h1:3txOjFJ8M/JFs01h7xOrnQHVn6hZgDNA16ivyUlu1iU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= From 43764772aa6f90fbeaf99cd6bccdebc157b5330a Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Thu, 10 Oct 2024 11:59:53 +0300 Subject: [PATCH 486/548] [#151] index page: Add browse via native protocol Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 20 ++- cmd/http-gw/settings.go | 6 +- config/config.env | 9 + config/config.yaml | 3 + docs/gate-configuration.md | 30 ++-- go.mod | 1 + go.sum | 2 + internal/handler/browse.go | 300 ++++++++++++++++++++++++++----- internal/handler/download.go | 23 +-- internal/handler/handler.go | 87 +++++---- internal/handler/handler_test.go | 12 +- internal/handler/head.go | 4 +- internal/handler/utils.go | 21 +-- internal/logs/logs.go | 6 +- internal/templates/index.gotmpl | 42 +++-- tree/tree.go | 104 ++++++++++- 16 files changed, 537 insertions(+), 133 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 853977c..0dd53a6 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -40,6 +40,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" + "github.com/panjf2000/ants/v2" "github.com/spf13/viper" "github.com/valyala/fasthttp" "go.opentelemetry.io/otel/trace" @@ -89,6 +90,7 @@ type ( appSettings struct { reconnectInterval time.Duration dialerSource *internalnet.DialerSource + workerPoolSize int mu sync.RWMutex defaultTimestamp bool @@ -184,6 +186,7 @@ func (a *app) initAppSettings() { a.settings = &appSettings{ reconnectInterval: fetchReconnectInterval(a.cfg), dialerSource: getDialerSource(a.log, a.cfg), + workerPoolSize: a.cfg.GetInt(cfgWorkerPoolSize), } a.settings.update(a.cfg, a.log) } @@ -490,7 +493,13 @@ func (a *app) setHealthStatus() { } func (a *app) Serve() { - handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool))) + workerPool := a.initWorkerPool() + defer func() { + workerPool.Release() + close(a.webDone) + }() + + handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) // Configure router. a.configureRouter(handler) @@ -532,8 +541,14 @@ LOOP: a.metrics.Shutdown() a.stopServices() a.shutdownTracing() +} - close(a.webDone) +func (a *app) initWorkerPool() *ants.Pool { + workerPool, err := ants.NewPool(a.settings.workerPoolSize) + if err != nil { + a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err)) + } + return workerPool } func (a *app) shutdownTracing() { @@ -609,6 +624,7 @@ func (a *app) stopServices() { svc.ShutDown(ctx) } } + func (a *app) configureRouter(handler *handler.Handler) { r := router.New() r.RedirectTrailingSlash = true diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 4f1712b..316c500 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -71,6 +71,8 @@ const ( cfgIndexPageEnabled = "index_page.enabled" cfgIndexPageTemplatePath = "index_page.template_path" + cfgWorkerPoolSize = "worker_pool_size" + // Web. cfgWebReadBufferSize = "web.read_buffer_size" cfgWebWriteBufferSize = "web.write_buffer_size" @@ -228,9 +230,6 @@ func settings() *viper.Viper { // pool: v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) - v.SetDefault(cfgIndexPageEnabled, false) - v.SetDefault(cfgIndexPageTemplatePath, "") - // frostfs: v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut) @@ -242,6 +241,7 @@ func settings() *viper.Viper { v.SetDefault(cfgWebStreamRequestBody, true) v.SetDefault(cfgWebMaxRequestBodySize, fasthttp.DefaultMaxRequestBodySize) + v.SetDefault(cfgWorkerPoolSize, 1000) // upload header v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) diff --git a/config/config.env b/config/config.env index db54c92..fd51392 100644 --- a/config/config.env +++ b/config/config.env @@ -150,3 +150,12 @@ HTTP_GW_MULTINET_FALLBACK_DELAY=300ms # List of subnets and IP addresses to use as source for those subnets HTTP_GW_MULTINET_SUBNETS_1_MASK=1.2.3.4/24 HTTP_GW_MULTINET_SUBNETS_1_SOURCE_IPS=1.2.3.4 1.2.3.5 + +# Number of workers in handler's worker pool +HTTP_GW_WORKER_POOL_SIZE=1000 + +# Index page +# Enable index page support +HTTP_GW_INDEX_PAGE_ENABLED=false +# Index page template path +HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl \ No newline at end of file diff --git a/config/config.yaml b/config/config.yaml index 6c89e78..ef5c529 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -113,6 +113,9 @@ request_timeout: 5s # Timeout to check node health during rebalance. rebalance_timer: 30s # Interval to check nodes health. pool_error_threshold: 100 # The number of errors on connection after which node is considered as unhealthy. +# Number of workers in handler's worker pool +worker_pool_size: 1000 + # Enable index page to see objects list for specified container and prefix index_page: enabled: false diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 5b5b018..c6cb617 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -75,18 +75,21 @@ request_timeout: 5s rebalance_timer: 30s pool_error_threshold: 100 reconnect_interval: 1m +worker_pool_size: 1000 + ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|------------------------|------------|---------------|---------------|-------------------------------------------------------------------------------------------------| -| `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. | -| `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. | -| `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. | -| `stream_timeout` | `duration` | | `10s` | Timeout for individual operations in streaming RPC. | -| `request_timeout` | `duration` | | `15s` | Timeout to check node health during rebalance. | -| `rebalance_timer` | `duration` | | `60s` | Interval to check node health. | -| `pool_error_threshold` | `uint32` | | `100` | The number of errors on connection after which node is considered as unhealthy. | -| `reconnect_interval` | `duration` | no | `1m` | Listeners reconnection interval. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|------------------------|------------|---------------|---------------|------------------------------------------------------------------------------------| +| `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. | +| `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. | +| `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. | +| `stream_timeout` | `duration` | | `10s` | Timeout for individual operations in streaming RPC. | +| `request_timeout` | `duration` | | `15s` | Timeout to check node health during rebalance. | +| `rebalance_timer` | `duration` | | `60s` | Interval to check node health. | +| `pool_error_threshold` | `uint32` | | `100` | The number of errors on connection after which node is considered as unhealthy. | +| `reconnect_interval` | `duration` | no | `1m` | Listeners reconnection interval. | +| `worker_pool_size` | `int` | no | `1000` | Maximum worker count in handler's worker pool. | # `wallet` section @@ -374,7 +377,12 @@ resolve_bucket: # `index_page` section -Parameters for index HTML-page output with S3-bucket or S3-subdir content for `Get object` request +Parameters for index HTML-page output. Activates if `GetObject` request returns `not found`. Two +index page modes available: + +* `s3` mode uses tree service for listing objects, +* `native` sends requests to nodes via native protocol. + If request pass S3-bucket name instead of CID, `s3` mode will be used, otherwise `native`. ```yaml index_page: diff --git a/go.mod b/go.mod index ca8f4fe..a2f41d8 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/docker/go-units v0.4.0 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.106.2 + github.com/panjf2000/ants/v2 v2.5.0 github.com/prometheus/client_golang v1.19.0 github.com/prometheus/client_model v0.5.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 197f21c..a7a5be4 100644 --- a/go.sum +++ b/go.sum @@ -682,6 +682,8 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q= +github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= diff --git a/internal/handler/browse.go b/internal/handler/browse.go index c89d8c5..b24a569 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -1,15 +1,21 @@ package handler import ( + "context" "html/template" "net/url" "sort" "strconv" "strings" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/docker/go-units" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -25,19 +31,68 @@ const ( type ( BrowsePageData struct { - BucketName, - Prefix string - Objects []ResponseObject + HasErrors bool + Container string + Prefix string + Protocol string + Objects []ResponseObject } ResponseObject struct { OID string Created string FileName string + FilePath string Size string IsDir bool + GetURL string } ) +func newListObjectsResponseS3(attrs map[string]string) ResponseObject { + return ResponseObject{ + Created: formatTimestamp(attrs[attrCreated]), + OID: attrs[attrOID], + FileName: attrs[attrFileName], + Size: attrs[attrSize], + IsDir: attrs[attrOID] == "", + } +} + +func newListObjectsResponseNative(attrs map[string]string) ResponseObject { + filename := lastPathElement(attrs[object.AttributeFilePath]) + if filename == "" { + filename = attrs[attrFileName] + } + return ResponseObject{ + OID: attrs[attrOID], + Created: formatTimestamp(attrs[object.AttributeTimestamp] + "000"), + FileName: filename, + FilePath: attrs[object.AttributeFilePath], + Size: attrs[attrSize], + IsDir: false, + } +} + +func getNextDir(filepath, prefix string) string { + restPath := strings.Replace(filepath, prefix, "", 1) + index := strings.Index(restPath, "/") + if index == -1 { + return "" + } + return restPath[:index] +} + +func lastPathElement(path string) string { + if path == "" { + return path + } + index := strings.LastIndex(path, "/") + if index == len(path)-1 { + index = strings.LastIndex(path[:index], "/") + } + return path[index+1:] +} + func parseTimestamp(tstamp string) (time.Time, error) { millis, err := strconv.ParseInt(tstamp, 10, 64) if err != nil { @@ -47,16 +102,6 @@ func parseTimestamp(tstamp string) (time.Time, error) { return time.UnixMilli(millis), nil } -func NewResponseObject(nodes map[string]string) ResponseObject { - return ResponseObject{ - OID: nodes[attrOID], - Created: nodes[attrCreated], - FileName: nodes[attrFileName], - Size: nodes[attrSize], - IsDir: nodes[attrOID] == "", - } -} - func formatTimestamp(strdate string) string { date, err := parseTimestamp(strdate) if err != nil || date.IsZero() { @@ -94,12 +139,9 @@ func trimPrefix(encPrefix string) string { return prefix[:slashIndex] } -func urlencode(prefix, filename string) string { +func urlencode(path string) string { var res strings.Builder - path := filename - if prefix != "" { - path = strings.Join([]string{prefix, filename}, "/") - } + prefixParts := strings.Split(path, "/") for _, prefixPart := range prefixParts { prefixPart = "/" + url.PathEscape(prefixPart) @@ -112,46 +154,220 @@ func urlencode(prefix, filename string) string { return res.String() } -func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketInfo, prefix string) { +type GetObjectsResponse struct { + objects []ResponseObject + hasErrors bool +} + +func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { + nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) + if err != nil { + return nil, err + } + + result := &GetObjectsResponse{ + objects: make([]ResponseObject, 0, len(nodes)), + } + for _, node := range nodes { + meta := node.GetMeta() + if meta == nil { + continue + } + var attrs = make(map[string]string, len(meta)) + for _, m := range meta { + attrs[m.GetKey()] = string(m.GetValue()) + } + obj := newListObjectsResponseS3(attrs) + obj.FilePath = prefix + obj.FileName + obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath) + result.objects = append(result.objects, obj) + } + + return result, nil +} + +func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { + var basePath string + if ind := strings.LastIndex(prefix, "/"); ind != -1 { + basePath = prefix[:ind+1] + } + + filters := object.NewSearchFilters() + filters.AddRootFilter() + if prefix != "" { + filters.AddFilter(object.AttributeFilePath, prefix, object.MatchCommonPrefix) + } + + prm := PrmObjectSearch{ + PrmAuth: PrmAuth{ + BearerToken: bearerToken(ctx), + }, + Container: bucketInfo.CID, + Filters: filters, + } + objectIDs, err := h.frostfs.SearchObjects(ctx, prm) + if err != nil { + return nil, err + } + defer objectIDs.Close() + + resp, err := h.headDirObjects(ctx, bucketInfo.CID, objectIDs, basePath) + if err != nil { + return nil, err + } + + log := utils.GetReqLogOrDefault(ctx, h.log) + dirs := make(map[string]struct{}) + result := &GetObjectsResponse{ + objects: make([]ResponseObject, 0, 100), + } + for objExt := range resp { + if objExt.Error != nil { + log.Error(logs.FailedToHeadObject, zap.Error(objExt.Error)) + result.hasErrors = true + continue + } + if objExt.Object.IsDir { + if _, ok := dirs[objExt.Object.FileName]; ok { + continue + } + objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + urlencode(objExt.Object.FilePath) + dirs[objExt.Object.FileName] = struct{}{} + } else { + objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + objExt.Object.OID + } + result.objects = append(result.objects, objExt.Object) + } + return result, nil +} + +type ResponseObjectExtended struct { + Object ResponseObject + Error error +} + +func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs ResObjectSearch, basePath string) (<-chan ResponseObjectExtended, error) { + res := make(chan ResponseObjectExtended) + + go func() { + defer close(res) + log := utils.GetReqLogOrDefault(ctx, h.log).With( + zap.String("cid", cnrID.EncodeToString()), + zap.String("path", basePath), + ) + var wg sync.WaitGroup + err := objectIDs.Iterate(func(id oid.ID) bool { + wg.Add(1) + err := h.workerPool.Submit(func() { + defer wg.Done() + var obj ResponseObjectExtended + obj.Object, obj.Error = h.headDirObject(ctx, cnrID, id, basePath) + res <- obj + }) + if err != nil { + wg.Done() + log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err)) + } + select { + case <-ctx.Done(): + return true + default: + return false + } + }) + if err != nil { + log.Error(logs.FailedToIterateOverResponse, zap.Error(err)) + } + wg.Wait() + }() + + return res, nil +} + +func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID, basePath string) (ResponseObject, error) { + addr := newAddress(cnrID, objID) + obj, err := h.frostfs.HeadObject(ctx, PrmObjectHead{ + PrmAuth: PrmAuth{BearerToken: bearerToken(ctx)}, + Address: addr, + }) + if err != nil { + return ResponseObject{}, err + } + + attrs := loadAttributes(obj.Attributes()) + attrs[attrOID] = objID.EncodeToString() + if multipartSize, ok := attrs[attributeMultipartObjectSize]; ok { + attrs[attrSize] = multipartSize + } else { + attrs[attrSize] = strconv.FormatUint(obj.PayloadSize(), 10) + } + + dirname := getNextDir(attrs[object.AttributeFilePath], basePath) + if dirname == "" { + return newListObjectsResponseNative(attrs), nil + } + + return ResponseObject{ + FileName: dirname, + FilePath: basePath + dirname, + IsDir: true, + }, nil +} + +type browseParams struct { + bucketInfo *data.BucketInfo + prefix string + isNative bool + listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) (*GetObjectsResponse, error) +} + +func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) { + const S3Protocol = "s3" + const FrostfsProtocol = "frostfs" + ctx := utils.GetContextFromRequest(c) reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With(zap.String("bucket", bucketInfo.Name)) - - nodes, err := h.listObjects(ctx, bucketInfo, prefix) + log := reqLog.With( + zap.String("bucket", p.bucketInfo.Name), + zap.String("container", p.bucketInfo.CID.EncodeToString()), + zap.String("prefix", p.prefix), + ) + resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix) if err != nil { logAndSendBucketError(c, log, err) return } - respObjects := make([]ResponseObject, len(nodes)) - - for i, node := range nodes { - respObjects[i] = NewResponseObject(node) - } - - sort.Slice(respObjects, func(i, j int) bool { - if respObjects[i].IsDir == respObjects[j].IsDir { - return respObjects[i].FileName < respObjects[j].FileName + objects := resp.objects + sort.Slice(objects, func(i, j int) bool { + if objects[i].IsDir == objects[j].IsDir { + return objects[i].FileName < objects[j].FileName } - return respObjects[i].IsDir + return objects[i].IsDir }) - indexTemplate := h.config.IndexPageTemplate() tmpl, err := template.New("index").Funcs(template.FuncMap{ - "formatTimestamp": formatTimestamp, - "formatSize": formatSize, - "trimPrefix": trimPrefix, - "urlencode": urlencode, - "parentDir": parentDir, - }).Parse(indexTemplate) + "formatSize": formatSize, + "trimPrefix": trimPrefix, + "urlencode": urlencode, + "parentDir": parentDir, + }).Parse(h.config.IndexPageTemplate()) if err != nil { logAndSendBucketError(c, log, err) return } + bucketName := p.bucketInfo.Name + protocol := S3Protocol + if p.isNative { + bucketName = p.bucketInfo.CID.EncodeToString() + protocol = FrostfsProtocol + } if err = tmpl.Execute(c, &BrowsePageData{ - BucketName: bucketInfo.Name, - Prefix: prefix, - Objects: respObjects, + Container: bucketName, + Prefix: p.prefix, + Objects: objects, + Protocol: protocol, + HasErrors: resp.hasErrors, }); err != nil { logAndSendBucketError(c, log, err) return diff --git a/internal/handler/download.go b/internal/handler/download.go index 19380d4..cd4e55a 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -23,13 +23,16 @@ import ( // DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { - test, _ := c.UserValue("oid").(string) - var id oid.ID - err := id.DecodeString(test) - if err != nil { - h.byObjectName(c, h.receiveFile) - } else { - h.byAddress(c, h.receiveFile) + oidURLParam := c.UserValue("oid").(string) + downloadQueryParam := c.QueryArgs().GetBool("download") + + switch { + case isObjectID(oidURLParam): + h.byNativeAddress(c, h.receiveFile) + case !isContainerRoot(oidURLParam) && (downloadQueryParam || !isDir(oidURLParam)): + h.byS3Path(c, h.receiveFile) + default: + h.browseIndex(c) } } @@ -45,7 +48,7 @@ func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { h.byAttribute(c, h.receiveFile) } -func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) { +func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) { filters := object.NewSearchFilters() filters.AddRootFilter() filters.AddFilter(key, val, op) @@ -54,7 +57,7 @@ func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op PrmAuth: PrmAuth{ BearerToken: bearerToken(ctx), }, - Container: *cnrID, + Container: cnrID, Filters: filters, } @@ -102,7 +105,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { return } - resSearch, err := h.search(ctx, &bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) + resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 62d0897..9ed7f99 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -22,6 +22,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/panjf2000/ants/v2" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -165,6 +166,7 @@ type Handler struct { containerResolver ContainerResolver tree *tree.Tree cache *cache.BucketCache + workerPool *ants.Pool } type AppParams struct { @@ -175,7 +177,7 @@ type AppParams struct { Cache *cache.BucketCache } -func New(params *AppParams, config Config, tree *tree.Tree) *Handler { +func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { return &Handler{ log: params.Logger, frostfs: params.FrostFS, @@ -184,14 +186,15 @@ func New(params *AppParams, config Config, tree *tree.Tree) *Handler { containerResolver: params.Resolver, tree: tree, cache: params.Cache, + workerPool: workerPool, } } -// byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that +// byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { +func (h *Handler) byNativeAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { idCnr, _ := c.UserValue("cid").(string) - idObj, _ := c.UserValue("oid").(string) + idObj, _ := url.PathUnescape(c.UserValue("oid").(string)) ctx := utils.GetContextFromRequest(c) reqLog := utils.GetReqLogOrDefault(ctx, h.log) @@ -215,12 +218,11 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ f(ctx, *h.newRequest(c, log), addr) } -// byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that -// prepares request and object address to it. -func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { +// byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that +// resolves object address from S3-like path /. +func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { bucketname := c.UserValue("cid").(string) key := c.UserValue("oid").(string) - download := c.QueryArgs().GetBool("download") ctx := utils.GetContextFromRequest(c) reqLog := utils.GetReqLogOrDefault(ctx, h.log) @@ -239,15 +241,6 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r } foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey) - if h.config.IndexPageEnabled() && !download && string(c.Method()) != fasthttp.MethodHead { - if isDir(unescapedKey) || isContainerRoot(unescapedKey) { - if code := checkErrorType(err); code == fasthttp.StatusNotFound || code == fasthttp.StatusOK { - c.SetStatusCode(code) - h.browseObjects(c, bktInfo, unescapedKey) - return - } - } - } if err != nil { if errors.Is(err, tree.ErrNodeAccessDenied) { response.Error(c, "Access Denied", fasthttp.StatusForbidden) @@ -267,7 +260,7 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r f(ctx, *h.newRequest(c, log), addr) } -// byAttribute is a wrapper similar to byAddress. +// byAttribute is a wrapper similar to byNativeAddress. func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { scid, _ := c.UserValue("cid").(string) key, _ := c.UserValue("attr_key").(string) @@ -298,7 +291,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re return } - res, err := h.search(ctx, &bktInfo.CID, key, val, object.MatchStringEqual) + res, err := h.search(ctx, bktInfo.CID, key, val, object.MatchStringEqual) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) @@ -395,24 +388,50 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } -func (h *Handler) listObjects(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]map[string]string, error) { - nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) +func (h *Handler) browseIndex(c *fasthttp.RequestCtx) { + if !h.config.IndexPageEnabled() { + c.SetStatusCode(fasthttp.StatusNotFound) + return + } + + cidURLParam := c.UserValue("cid").(string) + oidURLParam := c.UserValue("oid").(string) + + ctx := utils.GetContextFromRequest(c) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("cid", cidURLParam), zap.String("oid", oidURLParam)) + + unescapedKey, err := url.QueryUnescape(oidURLParam) if err != nil { - return nil, err + logAndSendBucketError(c, log, err) + return } - var objects = make([]map[string]string, 0, len(nodes)) - for _, node := range nodes { - meta := node.GetMeta() - if meta == nil { - continue - } - var obj = make(map[string]string, len(meta)) - for _, m := range meta { - obj[m.GetKey()] = string(m.GetValue()) - } - objects = append(objects, obj) + bktInfo, err := h.getBucketInfo(ctx, cidURLParam, log) + if err != nil { + logAndSendBucketError(c, log, err) + return } - return objects, nil + listFunc := h.getDirObjectsS3 + isNativeList := false + + err = h.tree.CheckSettingsNodeExist(ctx, bktInfo) + if err != nil { + if errors.Is(err, tree.ErrNodeNotFound) { + // tree probe failed, try to use native + listFunc = h.getDirObjectsNative + isNativeList = true + } else { + logAndSendBucketError(c, log, err) + return + } + } + + h.browseObjects(c, browseParams{ + bucketInfo: bktInfo, + prefix: unescapedKey, + listObjects: listFunc, + isNative: isNativeList, + }) } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 4fe9153..34668a5 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -26,6 +26,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -57,10 +58,11 @@ func (c *configMock) IndexPageEnabled() bool { return false } -func (c *configMock) IndexPageTemplatePath() string { +func (c *configMock) IndexPageTemplate() string { return "" } -func (c *configMock) IndexPageTemplate() string { + +func (c *configMock) IndexPageNativeTemplate() string { return "" } @@ -126,7 +128,11 @@ func prepareHandlerContext() (*handlerContext, error) { treeMock := &treeClientMock{} cfgMock := &configMock{} - handler := New(params, cfgMock, tree.NewTree(treeMock)) + workerPool, err := ants.NewPool(1000) + if err != nil { + return nil, err + } + handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool) return &handlerContext{ key: key, diff --git a/internal/handler/head.go b/internal/handler/head.go index f0a1e94..ccd6a91 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -107,9 +107,9 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { err := id.DecodeString(test) if err != nil { - h.byObjectName(c, h.headObject) + h.byS3Path(c, h.headObject) } else { - h.byAddress(c, h.headObject) + h.byNativeAddress(c, h.headObject) } } diff --git a/internal/handler/utils.go b/internal/handler/utils.go index a944b67..b537d64 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -2,17 +2,16 @@ package handler import ( "context" - "errors" "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -46,19 +45,21 @@ func isDir(name string) bool { return strings.HasSuffix(name, "/") } +func isObjectID(s string) bool { + var objID oid.ID + return objID.DecodeString(s) == nil +} + func isContainerRoot(key string) bool { return key == "" } -func checkErrorType(err error) int { - switch { - case err == nil: - return fasthttp.StatusOK - case errors.Is(err, tree.ErrNodeAccessDenied): - return fasthttp.StatusForbidden - default: - return fasthttp.StatusNotFound +func loadAttributes(attrs []object.Attribute) map[string]string { + result := make(map[string]string) + for _, attr := range attrs { + result[attr.Key()] = attr.Value() } + return result } func isValidToken(s string) bool { diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 409f87d..4dfa21f 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -31,7 +31,8 @@ const ( CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go - FailedToReadIndexPageTemplate = "failed to read index page template, set default" // Warn in ../../app.go + FailedToCreateWorkerPool = "failed to create worker pool" // Fatal in ../../app.go + FailedToReadIndexPageTemplate = "failed to read index page template" // Error in ../../app.go SetCustomIndexPageTemplate = "set custom index page template" // Info in ../../app.go ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go @@ -71,6 +72,9 @@ const ( AddedStoragePeer = "added storage peer" // Info in ../../settings.go CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go + FailedToSumbitTaskToPool = "failed to submit task to pool" // Error in ../handler/browse.go + FailedToHeadObject = "failed to head object" // Error in ../handler/browse.go + FailedToIterateOverResponse = "failed to iterate over search response" // Error in ../handler/browse.go InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go diff --git a/internal/templates/index.gotmpl b/internal/templates/index.gotmpl index ea66a62..b14cc06 100644 --- a/internal/templates/index.gotmpl +++ b/internal/templates/index.gotmpl @@ -1,11 +1,20 @@ -{{$bucketName := .BucketName}} +{{$container := .Container}} {{ $prefix := trimPrefix .Prefix }} - Index of s3://{{$bucketName}}/{{if $prefix}}/{{$prefix}}/{{end}} + Index of {{.Protocol}}://{{$container}} + /{{if $prefix}}/{{$prefix}}/{{end}} -

Index of s3://{{$bucketName}}/{{if $prefix}}{{$prefix}}/{{end}}

+

Index of {{.Protocol}}://{{$container}}/{{if $prefix}}{{$prefix}}/{{end}}

+{{ if .HasErrors }} +
+ Errors occurred while processing the request. Perhaps some objects are missing +
+{{ end }} + @@ -42,20 +61,22 @@ {{if $trimmedPrefix }} + {{else}} + {{end}} {{range .Objects}} @@ -63,21 +84,22 @@ + - + - {{ $trimmedPrefix := trimPrefix $prefix }} - {{if $trimmedPrefix }} + {{ $parentPrefix := getParent .Prefix }} + {{if $parentPrefix }} diff --git a/tree/tree.go b/tree/tree.go index 2ee9356..d99e24b 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -7,7 +7,6 @@ import ( "strings" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -52,10 +51,10 @@ type ( var ( // ErrNodeNotFound is returned from ServiceClient in case of not found error. - ErrNodeNotFound = layer.ErrNodeNotFound + ErrNodeNotFound = errors.New("not found") // ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error. - ErrNodeAccessDenied = layer.ErrNodeAccessDenied + ErrNodeAccessDenied = errors.New("access denied") ) const ( @@ -259,7 +258,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name nodes = filterMultipartNodes(nodes) if len(nodes) == 0 { - return nil, layer.ErrNodeNotFound + return nil, ErrNodeNotFound } if len(nodes) != 1 { c.reqLogger(ctx).Warn(logs.FoundSeveralSystemTreeNodes, zap.String("name", name), logs.TagField(logs.TagExternalStorageTree)) @@ -303,7 +302,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { } if targetIndexNode == -1 { - return nil, fmt.Errorf("latest version: %w", layer.ErrNodeNotFound) + return nil, fmt.Errorf("latest version: %w", ErrNodeNotFound) } return nodes[targetIndexNode], nil @@ -324,20 +323,23 @@ func pathFromName(objectName string) []string { return strings.Split(objectName, separator) } -func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) { +func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, error) { ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix") defer span.End() - rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix) + rootID, err := c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(prefix, separator)) if err != nil { - return nil, "", err + if errors.Is(err, ErrNodeNotFound) { + return nil, nil + } + return nil, err } subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false) if err != nil { if errors.Is(err, ErrNodeNotFound) { - return nil, "", nil + return nil, nil } - return nil, "", err + return nil, err } nodesMap := make(map[string][]NodeResponse, len(subTree)) @@ -347,10 +349,6 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, } fileName := GetFilename(node) - if !strings.HasPrefix(fileName, tailPrefix) { - continue - } - nodes := nodesMap[fileName] // Add all nodes if flag latestOnly is false. @@ -374,7 +372,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, result = append(result, nodeResponseToNodeInfo(nodes)...) } - return result, strings.TrimSuffix(prefix, tailPrefix), nil + return result, nil } func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo { @@ -386,22 +384,6 @@ func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo { return nodesInfo } -func (c *Tree) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) ([]uint64, string, error) { - rootID := []uint64{0} - path := strings.Split(prefix, separator) - tailPrefix := path[len(path)-1] - - if len(path) > 1 { - var err error - rootID, err = c.getPrefixNodeID(ctx, bktInfo, treeID, path[:len(path)-1]) - if err != nil { - return nil, "", err - } - } - - return rootID, tailPrefix, nil -} - func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefixPath []string) ([]uint64, error) { p := &GetNodesParams{ CnrID: bktInfo.CID, @@ -424,7 +406,7 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr } if len(intermediateNodes) == 0 { - return nil, layer.ErrNodeNotFound + return nil, ErrNodeNotFound } return intermediateNodes, nil From 96a22d98f206ce4910d69ce68da221802cb23c22 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 25 Apr 2025 10:03:16 +0300 Subject: [PATCH 548/548] [#232] Use contract to get container info Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 28 ++++++- cmd/http-gw/settings.go | 8 ++ config/config.env | 3 + config/config.yaml | 5 ++ docs/gate-configuration.md | 13 ++++ go.mod | 2 +- internal/handler/container.go | 42 +++++++++++ internal/handler/frostfs_mock.go | 10 +++ internal/handler/handler.go | 47 +++--------- internal/handler/handler_test.go | 2 +- internal/logs/logs.go | 6 +- .../service/contracts/container/client.go | 73 +++++++++++++++++++ internal/service/contracts/util/util.go | 34 +++++++++ 13 files changed, 229 insertions(+), 44 deletions(-) create mode 100644 internal/handler/container.go create mode 100644 internal/service/contracts/container/client.go create mode 100644 internal/service/contracts/util/util.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index f603d3b..4a83caf 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -22,6 +22,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" + containerClient "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/contracts/container" + contractsUtil "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/contracts/util" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" @@ -39,6 +41,7 @@ import ( "github.com/nspcc-dev/neo-go/cli/flags" "github.com/nspcc-dev/neo-go/cli/input" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/panjf2000/ants/v2" @@ -276,6 +279,14 @@ func (a *app) initContainers(ctx context.Context) { a.corsCnrID = *corsCnrID } +func (a *app) initRPCClient(ctx context.Context) *rpcclient.Client { + rpcCli, err := rpcclient.New(ctx, a.config().GetString(cfgRPCEndpoint), rpcclient.Options{}) + if err != nil { + a.log.Fatal(logs.InitRPCClientFailed, zap.Error(err), logs.TagField(logs.TagApp)) + } + return rpcCli +} + func (a *app) initAppSettings(lc *logLevelConfig) { a.settings = &appSettings{ reconnectInterval: fetchReconnectInterval(a.config()), @@ -750,7 +761,22 @@ func (a *app) stopServices() { } func (a *app) configureRouter(workerPool *ants.Pool) { - a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool) + rpcCli := a.initRPCClient(a.ctx) + cnrContractName := a.config().GetString(cfgContractsContainerName) + rpcEndpoint := a.config().GetString(cfgRPCEndpoint) + cnrAddr, err := contractsUtil.ResolveContractHash(cnrContractName, rpcEndpoint) + if err != nil { + a.log.Fatal(logs.FailedToResolveContractHash, zap.Error(err), logs.TagField(logs.TagApp)) + } + cnrClient, err := containerClient.New(containerClient.Config{ + ContractHash: cnrAddr, + Key: a.key, + RPCClient: rpcCli, + }) + if err != nil { + a.log.Fatal(logs.InitContainerContractFailed, zap.Error(err), logs.TagField(logs.TagApp)) + } + a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), cnrClient, workerPool) r := router.New() r.RedirectTrailingSlash = true diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 07722de..4071969 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -62,6 +62,8 @@ const ( defaultMultinetFallbackDelay = 300 * time.Millisecond + defaultContainerContractName = "container.frostfs" + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" @@ -197,6 +199,9 @@ const ( cmdConfig = "config" cmdConfigDir = "config-dir" cmdListenAddress = "listen_address" + + // Contracts. + cfgContractsContainerName = "contracts.container.name" ) var ignore = map[string]struct{}{ @@ -401,6 +406,9 @@ func setDefaults(v *viper.Viper, flags *pflag.FlagSet) { // multinet v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay) + // contracts + v.SetDefault(cfgContractsContainerName, defaultContainerContractName) + if resolveMethods, err := flags.GetStringSlice(cfgResolveOrder); err == nil { v.SetDefault(cfgResolveOrder, resolveMethods) } diff --git a/config/config.env b/config/config.env index a86f3e8..ff880d5 100644 --- a/config/config.env +++ b/config/config.env @@ -181,3 +181,6 @@ HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true # Containers properties HTTP_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj + +# Container contract hash (LE) or name in NNS. +HTTP_GW_CONTRACTS_CONTAINER_NAME=container.frostfs diff --git a/config/config.yaml b/config/config.yaml index bb01d47..9b4b3c9 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -199,3 +199,8 @@ features: containers: cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj + +contracts: + container: + # Container contract hash (LE) or name in NNS. + name: container.frostfs diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 08e2679..7f3c4ef 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -60,6 +60,7 @@ $ cat http.log | `multinet` | [Multinet configuration](#multinet-section) | | `features` | [Features configuration](#features-section) | | `containers` | [Containers configuration](#containers-section) | +| `contracts` | [Contracts configuration](#contracts-section) | # General section @@ -527,3 +528,15 @@ containers: | Parameter | Type | SIGHUP reload | Default value | Description | |-----------|----------|---------------|---------------|-----------------------------------------| | `cors` | `string` | no | | Container name for CORS configurations. | + +# `contracts` section + +```yaml +contracts: + container: + name: container.frostfs +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|------------------|----------|---------------|---------------------|----------------------------------------------| +| `container.name` | `string` | no | `container.frostfs` | Container contract hash (LE) or name in NNS. | diff --git a/go.mod b/go.mod index c065b57..6082ef6 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.23 require ( + git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc @@ -33,7 +34,6 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect - git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e // indirect git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect diff --git a/internal/handler/container.go b/internal/handler/container.go new file mode 100644 index 0000000..3c7bec8 --- /dev/null +++ b/internal/handler/container.go @@ -0,0 +1,42 @@ +package handler + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "go.uber.org/zap" +) + +func (h *Handler) containerInfo(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) { + info := &data.BucketInfo{ + CID: cnrID, + Name: cnrID.EncodeToString(), + } + res, err := h.cnrContract.GetContainerByID(cnrID) + if err != nil { + return nil, fmt.Errorf("get frostfs container: %w", err) + } + + cnr := *res + + if domain := container.ReadDomain(cnr); domain.Name() != "" { + info.Name = domain.Name() + info.Zone = domain.Zone() + } + info.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(cnr) + info.PlacementPolicy = cnr.PlacementPolicy() + + if err = h.cache.Put(info); err != nil { + h.reqLogger(ctx).Warn(logs.CouldntPutBucketIntoCache, + zap.String("bucket name", info.Name), + zap.Stringer("cid", info.CID), + zap.Error(err), + logs.TagField(logs.TagDatapath)) + } + + return info, nil +} diff --git a/internal/handler/frostfs_mock.go b/internal/handler/frostfs_mock.go index 7d72ad9..540697f 100644 --- a/internal/handler/frostfs_mock.go +++ b/internal/handler/frostfs_mock.go @@ -233,6 +233,16 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) (Res return &resObjectSearchMock{res: res}, nil } +func (t *TestFrostFS) GetContainerByID(cid cid.ID) (*container.Container, error) { + for k, v := range t.containers { + if k == cid.EncodeToString() { + return v, nil + } + } + + return nil, fmt.Errorf("container does not exist %s", cid) +} + func (t *TestFrostFS) InitMultiObjectReader(context.Context, PrmInitMultiObjectReader) (io.Reader, error) { return nil, nil } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4d1dc31..2efd71d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -167,12 +167,18 @@ type ContainerResolver interface { Resolve(ctx context.Context, zone, name string) (*cid.ID, error) } +type ContainerContract interface { + // GetContainerByID reads a container from contract by ID. + GetContainerByID(cid.ID) (*container.Container, error) +} + type Handler struct { log *zap.Logger frostfs FrostFS ownerID *user.ID config Config containerResolver ContainerResolver + cnrContract ContainerContract tree *tree.Tree cache *cache.BucketCache workerPool *ants.Pool @@ -190,7 +196,7 @@ type AppParams struct { CORSCache *cache.CORSCache } -func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { +func New(params *AppParams, config Config, tree *tree.Tree, rpcCli ContainerContract, workerPool *ants.Pool) *Handler { return &Handler{ log: params.Logger, frostfs: params.FrostFS, @@ -202,6 +208,7 @@ func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Poo workerPool: workerPool, corsCnrID: params.CORSCnrID, corsCache: params.CORSCache, + cnrContract: rpcCli, } } @@ -308,43 +315,7 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string) (*dat return nil, fmt.Errorf("resolve container: %w", err) } - bktInfo, err := h.readContainer(ctx, *cnrID) - if err != nil { - return nil, fmt.Errorf("read container: %w", err) - } - - if err = h.cache.Put(bktInfo); err != nil { - h.reqLogger(ctx).Warn(logs.CouldntPutBucketIntoCache, - zap.String("bucket name", bktInfo.Name), - zap.Stringer("bucket cid", bktInfo.CID), - zap.Error(err), - logs.TagField(logs.TagDatapath)) - } - - return bktInfo, nil -} - -func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) { - prm := PrmContainer{ContainerID: cnrID} - res, err := h.frostfs.Container(ctx, prm) - if err != nil { - return nil, fmt.Errorf("get frostfs container '%s': %w", cnrID.String(), err) - } - - bktInfo := &data.BucketInfo{ - CID: cnrID, - Name: cnrID.EncodeToString(), - } - - if domain := container.ReadDomain(*res); domain.Name() != "" { - bktInfo.Name = domain.Name() - bktInfo.Zone = domain.Zone() - } - - bktInfo.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(*res) - bktInfo.PlacementPolicy = res.PlacementPolicy() - - return bktInfo, err + return h.containerInfo(ctx, *cnrID) } type ListFunc func(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 622940e..6c715fe 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -156,7 +156,7 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { if err != nil { return nil, err } - handler := New(params, cfgMock, tree.NewTree(treeMock, logger), workerPool) + handler := New(params, cfgMock, tree.NewTree(treeMock, logger), testFrostFS, workerPool) return &handlerContext{ key: key, diff --git a/internal/logs/logs.go b/internal/logs/logs.go index e7d118f..86921dd 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -73,6 +73,9 @@ const ( FailedToReadIndexPageTemplate = "failed to read index page template" SetCustomIndexPageTemplate = "set custom index page template" CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info" + InitRPCClientFailed = "init rpc client faileds" + InitContainerContractFailed = "init container contract failed" + FailedToResolveContractHash = "failed to resolve contract hash" ) // Log messages with the "datapath" tag. @@ -107,9 +110,7 @@ const ( IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" FailedToGetBucketInfo = "could not get bucket info" FailedToSubmitTaskToPool = "failed to submit task to pool" - ObjectWasDeleted = "object was deleted" IndexWasDeleted = "index was deleted" - FailedToGetLatestVersionOfObject = "failed to get latest version of object" FailedToGetLatestVersionOfIndexObject = "failed to get latest version of index object" FailedToCheckIfSettingsNodeExist = "failed to check if settings node exists" FailedToListObjects = "failed to list objects" @@ -121,7 +122,6 @@ const ( FailedToGetObjectPayload = "failed to get object payload" FailedToFindObjectByAttribute = "failed to get find object by attribute" FailedToUnescapePath = "failed to unescape path" - InvalidOIDParam = "invalid oid param" CouldNotGetCORSConfiguration = "could not get cors configuration" EmptyOriginRequestHeader = "empty Origin request header" EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header" diff --git a/internal/service/contracts/container/client.go b/internal/service/contracts/container/client.go new file mode 100644 index 0000000..09455be --- /dev/null +++ b/internal/service/contracts/container/client.go @@ -0,0 +1,73 @@ +package container + +import ( + "fmt" + "strings" + + containercontract "git.frostfs.info/TrueCloudLab/frostfs-contract/container" + containerclient "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/container" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/rpcclient" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/wallet" +) + +type Client struct { + contract *containerclient.Contract +} + +type Config struct { + ContractHash util.Uint160 + Key *keys.PrivateKey + RPCClient *rpcclient.Client +} + +func New(cfg Config) (*Client, error) { + var err error + key := cfg.Key + if key == nil { + if key, err = keys.NewPrivateKey(); err != nil { + return nil, fmt.Errorf("generate anon private key for container contract: %w", err) + } + } + acc := wallet.NewAccountFromPrivateKey(key) + + act, err := actor.NewSimple(cfg.RPCClient, acc) + if err != nil { + return nil, fmt.Errorf("create new actor: %w", err) + } + + return &Client{ + contract: containerclient.New(act, cfg.ContractHash), + }, nil +} + +func (c *Client) GetContainerByID(cnrID cid.ID) (*container.Container, error) { + items, err := c.contract.Get(cnrID[:]) + if err != nil { + if strings.Contains(err.Error(), containercontract.NotFoundError) { + return nil, fmt.Errorf("%w: %s", handler.ErrContainerNotFound, err) + } + return nil, err + } + + if len(items) != 4 { + return nil, fmt.Errorf("unexpected container stack item count: %d", len(items)) + } + + cnrBytes, err := items[0].TryBytes() + if err != nil { + return nil, fmt.Errorf("could not get byte array of container: %w", err) + } + + var cnr container.Container + if err = cnr.Unmarshal(cnrBytes); err != nil { + return nil, fmt.Errorf("can't unmarshal container: %w", err) + } + + return &cnr, nil +} diff --git a/internal/service/contracts/util/util.go b/internal/service/contracts/util/util.go new file mode 100644 index 0000000..444504b --- /dev/null +++ b/internal/service/contracts/util/util.go @@ -0,0 +1,34 @@ +package util + +import ( + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns" + "github.com/nspcc-dev/neo-go/pkg/util" +) + +// ResolveContractHash determine contract hash by resolving NNS name. +func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error) { + if hash, err := util.Uint160DecodeStringLE(contractHash); err == nil { + return hash, nil + } + + splitName := strings.Split(contractHash, ".") + if len(splitName) != 2 { + return util.Uint160{}, fmt.Errorf("invalid contract name: '%s'", contractHash) + } + + var domain container.Domain + domain.SetName(splitName[0]) + domain.SetZone(splitName[1]) + + var nns ns.NNS + if err := nns.Dial(rpcAddress); err != nil { + return util.Uint160{}, fmt.Errorf("dial nns %s: %w", rpcAddress, err) + } + defer nns.Close() + + return nns.ResolveContractHash(domain) +}
FilenameOID Size Created Download
- ⮐.. + ⮐..
- ⮐.. + ⮐..
{{if .IsDir}} 🗀 - + {{.FileName}}/ {{else}} 🗎 - + {{.FileName}} {{end}} {{.OID}} {{if not .IsDir}}{{ formatSize .Size }}{{end}}{{if not .IsDir}}{{ formatTimestamp .Created }}{{end}}{{ .Created }} - {{ if not .IsDir }} - + {{ if .OID }} + Link {{ end }} diff --git a/tree/tree.go b/tree/tree.go index 162f41f..40209a5 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -30,6 +30,11 @@ type ( Meta map[string]string } + multiSystemNode struct { + // the first element is latest + nodes []*treeNode + } + GetNodesParams struct { CnrID cid.ID BktInfo *data.BucketInfo @@ -50,18 +55,19 @@ var ( ) const ( - FileNameKey = "FileName" -) + FileNameKey = "FileName" + settingsFileName = "bucket-settings" -const ( - oidKV = "OID" + oidKV = "OID" + uploadIDKV = "UploadId" + sizeKV = "Size" // keys for delete marker nodes. isDeleteMarkerKV = "IsDeleteMarker" - sizeKV = "Size" // versionTree -- ID of a tree with object versions. versionTree = "version" + systemTree = "system" separator = "/" ) @@ -135,6 +141,45 @@ func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { return version } +func newMultiNode(nodes []NodeResponse) (*multiSystemNode, error) { + var ( + err error + index int + maxTimestamp uint64 + ) + + if len(nodes) == 0 { + return nil, errors.New("multi node must have at least one node") + } + + treeNodes := make([]*treeNode, len(nodes)) + + for i, node := range nodes { + if treeNodes[i], err = newTreeNode(node); err != nil { + return nil, fmt.Errorf("parse system node response: %w", err) + } + + if timestamp := getMaxTimestamp(node); timestamp > maxTimestamp { + index = i + maxTimestamp = timestamp + } + } + + treeNodes[0], treeNodes[index] = treeNodes[index], treeNodes[0] + + return &multiSystemNode{ + nodes: treeNodes, + }, nil +} + +func (m *multiSystemNode) Latest() *treeNode { + return m.nodes[0] +} + +func (m *multiSystemNode) Old() []*treeNode { + return m.nodes[1:] +} + func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { nodes, err := c.GetVersions(ctx, cnrID, objectName) if err != nil { @@ -165,6 +210,55 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string return c.service.GetNodes(ctx, p) } +func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error { + _, err := c.getSystemNode(ctx, bktInfo, settingsFileName) + if err != nil { + return err + } + + return nil +} + +func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) { + p := &GetNodesParams{ + CnrID: bktInfo.CID, + BktInfo: bktInfo, + TreeID: systemTree, + Path: []string{name}, + LatestOnly: false, + AllAttrs: true, + } + nodes, err := c.service.GetNodes(ctx, p) + if err != nil { + return nil, err + } + + nodes = filterMultipartNodes(nodes) + + if len(nodes) == 0 { + return nil, ErrNodeNotFound + } + + return newMultiNode(nodes) +} + +func filterMultipartNodes(nodes []NodeResponse) []NodeResponse { + res := make([]NodeResponse, 0, len(nodes)) + +LOOP: + for _, node := range nodes { + for _, meta := range node.GetMeta() { + if meta.GetKey() == uploadIDKV { + continue LOOP + } + } + + res = append(res, node) + } + + return res +} + func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { var ( maxCreationTime uint64 From a2f8cb673539c350e2b4c147c1510078fa7bd59e Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 20 Nov 2024 11:09:31 +0300 Subject: [PATCH 487/548] Release v0.31.0 Signed-off-by: Alex Vanin --- CHANGELOG.md | 52 +++++++++++++++++++++++++++++++++++++++++++++++----- VERSION | 2 +- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0990b51..dc422d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,51 @@ This document outlines major changes between releases. ## [Unreleased] +## [0.31.0] - Rongbuk - 2024-11-20 + +### Fixed +- Docker warnings during image build (#126) +- `trace_id` parameter in logs (#148) +- SIGHUP support for `tracing.enabled` config parameter (#157) + ### Added -- Support percent-encoding for GET queries (#134) -- Add `trace_id` to logs (#148) -- Add `cors` config params (#158) +- Vulnerability report document (#123) +- Root CA configuration for tracing (#139) +- Log sampling policy configuration (#147) +- Index page support for buckets and containers (#137, #151) +- CORS support (#158) +- Source IP binding configuration for FrostFS requests (#160) +- Tracing attributes (#164) ### Changed -- Update go version to 1.22 (#132) +- Updated Go version to 1.22 (#132) + +### Removed +- Duplicated NNS Resolver code (#129) + +## [0.30.3] - 2024-10-18 + +### Fixed +- Get response on S3 multipart object (#142) + +### Added +- Support percent-encoding for GET queries (#134) + +### Changed +- Split `FrostFS` interface into separate read methods (#127) + +## [0.30.2] - 2024-09-03 + +### Added +- Fuzzing tests (#135) + +## [0.30.1] - 2024-08-20 + +### Fixed +- Error counting in pool component before connection switch (#131) + +### Added +- Log of endpoint address during tree pool errors (#131) ## [0.30.0] - Kangshung - 2024-07-22 @@ -128,4 +166,8 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.28.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.0...v0.28.1 [0.29.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.1...v0.29.0 [0.30.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.29.0...v0.30.0 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.0...master +[0.30.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.0...v0.30.1 +[0.30.2]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.1...v0.30.2 +[0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.2...v0.30.3 +[0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.3...v0.31.0 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...master \ No newline at end of file diff --git a/VERSION b/VERSION index 9388ecb..7021025 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.30.0 +v0.31.0 From e81f01c2abfb0958da757065c3d795a7b11e14bb Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Sun, 24 Nov 2024 13:32:40 +0300 Subject: [PATCH 488/548] [#150] Add dropped logs metric Signed-off-by: Pavel Pogodaev --- cmd/http-gw/app.go | 92 ++++++++++++++++----------------- cmd/http-gw/integration_test.go | 4 +- cmd/http-gw/main.go | 3 +- cmd/http-gw/settings.go | 52 +++++++++++-------- metrics/desc.go | 18 +++++++ metrics/metrics.go | 69 ++++++++++++++++++------- 6 files changed, 147 insertions(+), 91 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 0dd53a6..6ac9b1c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -50,35 +50,38 @@ import ( type ( app struct { - ctx context.Context - log *zap.Logger - logLevel zap.AtomicLevel - pool *pool.Pool - treePool *treepool.Pool - key *keys.PrivateKey - owner *user.ID - cfg *viper.Viper - webServer *fasthttp.Server - webDone chan struct{} - resolver *resolver.ContainerResolver - metrics *gateMetrics - services []*metrics.Service - settings *appSettings + ctx context.Context + log *zap.Logger + logLevel zap.AtomicLevel + pool *pool.Pool + treePool *treepool.Pool + key *keys.PrivateKey + owner *user.ID + cfg *viper.Viper + webServer *fasthttp.Server + webDone chan struct{} + resolver *resolver.ContainerResolver + metrics *gateMetrics + services []*metrics.Service + settings *appSettings + loggerSettings *loggerSettings servers []Server unbindServers []ServerInfo mu sync.RWMutex } + loggerSettings struct { + mu sync.RWMutex + appMetrics *metrics.GateMetrics + } + // App is an interface for the main gateway function. App interface { Wait() Serve() } - // Option is an application option. - Option func(a *app) - gateMetrics struct { logger *zap.Logger provider *metrics.GateMetrics @@ -119,37 +122,17 @@ type ( } ) -// WithLogger returns Option to set a specific logger. -func WithLogger(l *zap.Logger, lvl zap.AtomicLevel) Option { - return func(a *app) { - if l == nil { - return - } - a.log = l - a.logLevel = lvl - } -} +func newApp(ctx context.Context, v *viper.Viper) App { + logSettings := &loggerSettings{} + log := pickLogger(v, logSettings) -// WithConfig returns Option to use specific Viper configuration. -func WithConfig(c *viper.Viper) Option { - return func(a *app) { - if c == nil { - return - } - a.cfg = c - } -} - -func newApp(ctx context.Context, opt ...Option) App { a := &app{ - ctx: ctx, - log: zap.L(), - cfg: viper.GetViper(), - webServer: new(fasthttp.Server), - webDone: make(chan struct{}), - } - for i := range opt { - opt[i](a) + ctx: ctx, + log: log.logger, + cfg: v, + loggerSettings: logSettings, + webServer: new(fasthttp.Server), + webDone: make(chan struct{}), } a.initAppSettings() @@ -227,6 +210,22 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { s.corsMaxAge = corsMaxAge } +func (s *loggerSettings) DroppedLogsInc() { + s.mu.RLock() + defer s.mu.RUnlock() + + if s.appMetrics != nil { + s.appMetrics.DroppedLogsInc() + } +} + +func (s *loggerSettings) setMetrics(appMetrics *metrics.GateMetrics) { + s.mu.Lock() + defer s.mu.Unlock() + + s.appMetrics = appMetrics +} + func (s *appSettings) DefaultTimestamp() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -338,6 +337,7 @@ func (a *app) initMetrics() { gateMetricsProvider := metrics.NewGateMetrics(a.pool) a.metrics = newGateMetrics(a.log, gateMetricsProvider, a.cfg.GetBool(cfgPrometheusEnabled)) a.metrics.SetHealth(metrics.HealthStatusStarting) + a.loggerSettings.setMetrics(a.metrics.provider) } func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics { diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 79a2da5..1516f1d 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -34,7 +34,6 @@ import ( "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" - "go.uber.org/zap/zapcore" ) type putResponse struct { @@ -102,8 +101,7 @@ func runServer(pathToWallet string) (App, context.CancelFunc) { v.Set(cfgWalletPath, pathToWallet) v.Set(cfgWalletPassphrase, "") - l, lvl := newStdoutLogger(v, zapcore.DebugLevel) - application := newApp(cancelCtx, WithConfig(v), WithLogger(l, lvl)) + application := newApp(cancelCtx, v) go application.Serve() return application, cancel diff --git a/cmd/http-gw/main.go b/cmd/http-gw/main.go index ea9fbd7..fdd148c 100644 --- a/cmd/http-gw/main.go +++ b/cmd/http-gw/main.go @@ -9,9 +9,8 @@ import ( func main() { globalContext, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) v := settings() - logger, atomicLevel := pickLogger(v) - application := newApp(globalContext, WithLogger(logger, atomicLevel), WithConfig(v)) + application := newApp(globalContext, v) go application.Serve() application.Wait() } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 316c500..2298124 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -182,6 +182,11 @@ var ignore = map[string]struct{}{ cmdVersion: {}, } +type Logger struct { + logger *zap.Logger + lvl zap.AtomicLevel +} + func settings() *viper.Viper { v := viper.New() v.AutomaticEnv() @@ -418,7 +423,11 @@ func mergeConfig(v *viper.Viper, fileName string) error { return v.MergeConfig(cfgFile) } -func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { +type LoggerAppSettings interface { + DroppedLogsInc() +} + +func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger { lvl, err := getLogLevel(v) if err != nil { panic(err) @@ -428,9 +437,9 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { switch dest { case destinationStdout: - return newStdoutLogger(v, lvl) + return newStdoutLogger(v, lvl, settings) case destinationJournald: - return newJournaldLogger(v, lvl) + return newJournaldLogger(v, lvl, settings) default: panic(fmt.Sprintf("wrong destination for logger: %s", dest)) } @@ -447,18 +456,20 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { // Logger records a stack trace for all messages at or above fatal level. // // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. -func newStdoutLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { +func newStdoutLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger { stdout := zapcore.AddSync(os.Stderr) level := zap.NewAtomicLevelAt(lvl) consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level) - consoleOutCore = samplingEnabling(v, consoleOutCore) + consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, settings) - l := zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) - return l, level + return &Logger{ + logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), + lvl: level, + } } -func newJournaldLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { +func newJournaldLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger { level := zap.NewAtomicLevelAt(lvl) encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields) @@ -470,11 +481,12 @@ func newJournaldLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.Atom zapjournald.SyslogPid(), }) - coreWithContext = samplingEnabling(v, coreWithContext) + coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, settings) - l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) - - return l, level + return &Logger{ + logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), + lvl: level, + } } func newLogEncoder() zapcore.Encoder { @@ -484,19 +496,17 @@ func newLogEncoder() zapcore.Encoder { return zapcore.NewConsoleEncoder(c) } -func samplingEnabling(v *viper.Viper, core zapcore.Core) zapcore.Core { - // Zap samples by logging the first cgfLoggerSamplingInitial entries with a given level - // and message within the specified time interval. - // In the above config, only the first cgfLoggerSamplingInitial log entries with the same level and message - // are recorded in cfgLoggerSamplingInterval interval. Every other log entry will be dropped within the interval since - // cfgLoggerSamplingThereafter is specified here. +func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, settings LoggerAppSettings) zapcore.Core { if v.GetBool(cfgLoggerSamplingEnabled) { - core = zapcore.NewSamplerWithOptions( - core, + core = zapcore.NewSamplerWithOptions(core, v.GetDuration(cfgLoggerSamplingInterval), v.GetInt(cfgLoggerSamplingInitial), v.GetInt(cfgLoggerSamplingThereafter), - ) + zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) { + if dec&zapcore.LogDropped > 0 { + settings.DroppedLogsInc() + } + })) } return core diff --git a/metrics/desc.go b/metrics/desc.go index e10050c..a00ab3e 100644 --- a/metrics/desc.go +++ b/metrics/desc.go @@ -76,6 +76,15 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"endpoint"}, }, }, + statisticSubsystem: { + droppedLogs: Description{ + Type: dto.MetricType_COUNTER, + Namespace: namespace, + Subsystem: statisticSubsystem, + Name: droppedLogs, + Help: "Dropped logs (by sampling) count", + }, + }, } type Description struct { @@ -148,3 +157,12 @@ func mustNewGaugeVec(description Description) *prometheus.GaugeVec { description.VariableLabels, ) } + +func mustNewCounter(description Description) prometheus.Counter { + if description.Type != dto.MetricType_COUNTER { + panic("invalid metric type") + } + return prometheus.NewCounter( + prometheus.CounterOpts(newOpts(description)), + ) +} diff --git a/metrics/metrics.go b/metrics/metrics.go index b516477..1c06868 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -10,15 +10,17 @@ import ( ) const ( - namespace = "frostfs_http_gw" - stateSubsystem = "state" - poolSubsystem = "pool" - serverSubsystem = "server" + namespace = "frostfs_http_gw" + stateSubsystem = "state" + poolSubsystem = "pool" + serverSubsystem = "server" + statisticSubsystem = "statistic" ) const ( healthMetric = "health" versionInfoMetric = "version_info" + droppedLogs = "dropped_logs" ) const ( @@ -30,21 +32,19 @@ const ( ) const ( - methodGetBalance = "get_balance" - methodPutContainer = "put_container" - methodGetContainer = "get_container" - methodListContainer = "list_container" - methodDeleteContainer = "delete_container" - methodGetContainerEacl = "get_container_eacl" - methodSetContainerEacl = "set_container_eacl" - methodEndpointInfo = "endpoint_info" - methodNetworkInfo = "network_info" - methodPutObject = "put_object" - methodDeleteObject = "delete_object" - methodGetObject = "get_object" - methodHeadObject = "head_object" - methodRangeObject = "range_object" - methodCreateSession = "create_session" + methodGetBalance = "get_balance" + methodPutContainer = "put_container" + methodGetContainer = "get_container" + methodListContainer = "list_container" + methodDeleteContainer = "delete_container" + methodEndpointInfo = "endpoint_info" + methodNetworkInfo = "network_info" + methodPutObject = "put_object" + methodDeleteObject = "delete_object" + methodGetObject = "get_object" + methodHeadObject = "head_object" + methodRangeObject = "range_object" + methodCreateSession = "create_session" ) // HealthStatus of the gate application. @@ -69,6 +69,7 @@ type GateMetrics struct { stateMetrics poolMetricsCollector serverMetrics + statisticMetrics } type stateMetrics struct { @@ -76,6 +77,10 @@ type stateMetrics struct { versionInfo *prometheus.GaugeVec } +type statisticMetrics struct { + droppedLogs prometheus.Counter +} + type poolMetricsCollector struct { scraper StatisticScraper overallErrors prometheus.Gauge @@ -96,10 +101,14 @@ func NewGateMetrics(p StatisticScraper) *GateMetrics { serverMetric := newServerMetrics() serverMetric.register() + statsMetric := newStatisticMetrics() + statsMetric.register() + return &GateMetrics{ stateMetrics: *stateMetric, poolMetricsCollector: *poolMetric, serverMetrics: *serverMetric, + statisticMetrics: *statsMetric, } } @@ -107,6 +116,7 @@ func (g *GateMetrics) Unregister() { g.stateMetrics.unregister() prometheus.Unregister(&g.poolMetricsCollector) g.serverMetrics.unregister() + g.statisticMetrics.unregister() } func newStateMetrics() *stateMetrics { @@ -116,6 +126,20 @@ func newStateMetrics() *stateMetrics { } } +func newStatisticMetrics() *statisticMetrics { + return &statisticMetrics{ + droppedLogs: mustNewCounter(appMetricsDesc[statisticSubsystem][droppedLogs]), + } +} + +func (s *statisticMetrics) register() { + prometheus.MustRegister(s.droppedLogs) +} + +func (s *statisticMetrics) unregister() { + prometheus.Unregister(s.droppedLogs) +} + func (m stateMetrics) register() { prometheus.MustRegister(m.healthCheck) prometheus.MustRegister(m.versionInfo) @@ -134,6 +158,13 @@ func (m stateMetrics) SetVersion(ver string) { m.versionInfo.WithLabelValues(ver).Set(1) } +func (s *statisticMetrics) DroppedLogsInc() { + if s == nil { + return + } + s.droppedLogs.Inc() +} + func newPoolMetricsCollector(p StatisticScraper) *poolMetricsCollector { return &poolMetricsCollector{ scraper: p, From b9e44c603dc2a4605de5119238e01aeb2086f9a7 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Mon, 9 Dec 2024 14:45:24 +0300 Subject: [PATCH 489/548] [#178] Update frostfs-sdk-go with new tree service client Add tree service's GetBucketSettings to use them to check for protocol to use (S3 or native). Also add mock implementations for this and GetLatestVersion methods. Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 2 +- cmd/http-gw/integration_test.go | 2 +- go.mod | 3 +-- go.sum | 6 ++---- internal/service/frostfs/pool_wrapper.go | 22 +++++++++++----------- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 6ac9b1c..53c001c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -16,7 +16,6 @@ import ( "syscall" "time" - v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" @@ -31,6 +30,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 1516f1d..dd4d60f 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -17,7 +17,7 @@ import ( "testing" "time" - containerv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" + containerv2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" diff --git a/go.mod b/go.mod index a2f41d8..3dd27b8 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,8 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.22 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 diff --git a/go.sum b/go.sum index a7a5be4..7f43c86 100644 --- a/go.sum +++ b/go.sum @@ -37,16 +37,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 h1:ivcdxQeQDnx4srF2ezoaeVlF0FAycSAztwfIUJnUI4s= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 h1:f7jan6eBDN88DKnKj8GKyWpfjBbSzjDALcDejYKRgCs= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3/go.mod h1:3txOjFJ8M/JFs01h7xOrnQHVn6hZgDNA16ivyUlu1iU= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d h1:FpXI+mOrmJk3t2MKQFZuhLjCHDyDeo5rtP1WXl7gUWc= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= diff --git a/internal/service/frostfs/pool_wrapper.go b/internal/service/frostfs/pool_wrapper.go index b978d73..f6be05f 100644 --- a/internal/service/frostfs/pool_wrapper.go +++ b/internal/service/frostfs/pool_wrapper.go @@ -9,20 +9,20 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" + apitree "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/tree" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" - grpcService "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree/service" ) type GetNodeByPathResponseInfoWrapper struct { - response *grpcService.GetNodeByPathResponse_Info + response *apitree.GetNodeByPathResponseInfo } func (n GetNodeByPathResponseInfoWrapper) GetNodeID() []uint64 { - return []uint64{n.response.GetNodeId()} + return []uint64{n.response.GetNodeID()} } func (n GetNodeByPathResponseInfoWrapper) GetParentID() []uint64 { - return []uint64{n.response.GetParentId()} + return []uint64{n.response.GetParentID()} } func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() []uint64 { @@ -30,8 +30,8 @@ func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() []uint64 { } func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { - res := make([]tree.Meta, len(n.response.Meta)) - for i, value := range n.response.Meta { + res := make([]tree.Meta, len(n.response.GetMeta())) + for i, value := range n.response.GetMeta() { res[i] = value } return res @@ -133,15 +133,15 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, } type GetSubTreeResponseBodyWrapper struct { - response *grpcService.GetSubTreeResponse_Body + response *apitree.GetSubTreeResponseBody } func (n GetSubTreeResponseBodyWrapper) GetNodeID() []uint64 { - return n.response.GetNodeId() + return n.response.GetNodeID() } func (n GetSubTreeResponseBodyWrapper) GetParentID() []uint64 { - resp := n.response.GetParentId() + resp := n.response.GetParentID() if resp == nil { // storage sends nil that should be interpreted as []uint64{0} // due to protobuf compatibility, see 'GetSubTree' function @@ -155,8 +155,8 @@ func (n GetSubTreeResponseBodyWrapper) GetTimestamp() []uint64 { } func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta { - res := make([]tree.Meta, len(n.response.Meta)) - for i, value := range n.response.Meta { + res := make([]tree.Meta, len(n.response.GetMeta())) + for i, value := range n.response.GetMeta() { res[i] = value } return res From bbc7c7367d3146e8e1bbc13d60f08022b0d6fcc1 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 10 Dec 2024 15:42:13 +0300 Subject: [PATCH 490/548] [#179] Refine CODEOWNERS settings Signed-off-by: Vitaliy Potyarkin --- CODEOWNERS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 43df11e..de5e48e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1,3 @@ -.* @alexvanin @dkirillov +.* @TrueCloudLab/storage-services-developers @TrueCloudLab/storage-services-committers +.forgejo/.* @potyarkin +Makefile @potyarkin From dc100f03a6a164b9c49fee87bf663670c114ca5d Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 12 Dec 2024 09:28:22 +0300 Subject: [PATCH 491/548] [#174] Add fallback path to search Fallback path to search is needed because some software may keep FileName attribute and ignore FilePath attribute during file upload. Therefore, if this feature is enabled under certain conditions (for more information, see gate-configuration.md) a search will be performed for the FileName attribute. Signed-off-by: Roman Loginov --- cmd/http-gw/app.go | 39 ++++--- cmd/http-gw/settings.go | 3 + config/config.env | 5 +- config/config.yaml | 4 + docs/gate-configuration.md | 15 ++- internal/handler/browse.go | 1 + internal/handler/handler.go | 52 +++++++--- internal/handler/handler_test.go | 171 ++++++++++++++++++++++++++++++- internal/logs/logs.go | 1 + 9 files changed, 256 insertions(+), 35 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 53c001c..9adaec2 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -95,21 +95,22 @@ type ( dialerSource *internalnet.DialerSource workerPoolSize int - mu sync.RWMutex - defaultTimestamp bool - zipCompression bool - clientCut bool - returnIndexPage bool - indexPageTemplate string - bufferMaxSizeForPut uint64 - namespaceHeader string - defaultNamespaces []string - corsAllowOrigin string - corsAllowMethods []string - corsAllowHeaders []string - corsExposeHeaders []string - corsAllowCredentials bool - corsMaxAge int + mu sync.RWMutex + defaultTimestamp bool + zipCompression bool + clientCut bool + returnIndexPage bool + indexPageTemplate string + bufferMaxSizeForPut uint64 + namespaceHeader string + defaultNamespaces []string + corsAllowOrigin string + corsAllowMethods []string + corsAllowHeaders []string + corsExposeHeaders []string + corsAllowCredentials bool + corsMaxAge int + enableFilepathFallback bool } CORS struct { @@ -189,6 +190,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { corsExposeHeaders := v.GetStringSlice(cfgCORSExposeHeaders) corsAllowCredentials := v.GetBool(cfgCORSAllowCredentials) corsMaxAge := fetchCORSMaxAge(v) + enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback) s.mu.Lock() defer s.mu.Unlock() @@ -208,6 +210,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { s.corsExposeHeaders = corsExposeHeaders s.corsAllowCredentials = corsAllowCredentials s.corsMaxAge = corsMaxAge + s.enableFilepathFallback = enableFilepathFallback } func (s *loggerSettings) DroppedLogsInc() { @@ -305,6 +308,12 @@ func (s *appSettings) FormContainerZone(ns string) (zone string, isDefault bool) return ns + ".ns", false } +func (s *appSettings) EnableFilepathFallback() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.enableFilepathFallback +} + func (a *app) initResolver() { var err error a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 2298124..62ef83e 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -164,6 +164,9 @@ const ( cfgMultinetFallbackDelay = "multinet.fallback_delay" cfgMultinetSubnets = "multinet.subnets" + // Feature. + cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" + // Command line args. cmdHelp = "help" cmdVersion = "version" diff --git a/config/config.env b/config/config.env index fd51392..2822357 100644 --- a/config/config.env +++ b/config/config.env @@ -158,4 +158,7 @@ HTTP_GW_WORKER_POOL_SIZE=1000 # Enable index page support HTTP_GW_INDEX_PAGE_ENABLED=false # Index page template path -HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl \ No newline at end of file +HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl + +# Enable using fallback path to search for a object by attribute +HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false diff --git a/config/config.yaml b/config/config.yaml index ef5c529..6296bd9 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -172,3 +172,7 @@ multinet: source_ips: - 1.2.3.4 - 1.2.3.5 + +features: + # Enable using fallback path to search for a object by attribute + enable_filepath_fallback: false diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index c6cb617..7476f5d 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -59,7 +59,7 @@ $ cat http.log | `resolve_bucket` | [Bucket name resolving configuration](#resolve_bucket-section) | | `index_page` | [Index page configuration](#index_page-section) | | `multinet` | [Multinet configuration](#multinet-section) | - +| `features` | [Features configuration](#features-section) | # General section @@ -457,3 +457,16 @@ multinet: |--------------|------------|---------------|---------------|----------------------------------------------------------------------| | `mask` | `string` | yes | | Destination subnet. | | `source_ips` | `[]string` | yes | | Array of source IP addresses to use when dialing destination subnet. | + +# `features` section + +Contains parameters for enabling features. + +```yaml +features: + enable_filepath_fallback: true +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +| ----------------------------------- | ------ | ------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. | diff --git a/internal/handler/browse.go b/internal/handler/browse.go index b24a569..c54ab76 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -26,6 +26,7 @@ const ( attrOID = "OID" attrCreated = "Created" attrFileName = "FileName" + attrFilePath = "FilePath" attrSize = "Size" ) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9ed7f99..1839bf0 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -35,6 +35,7 @@ type Config interface { IndexPageTemplate() string BufferMaxSizeForPut() uint64 NamespaceHeader() string + EnableFilepathFallback() bool } // PrmContainer groups parameters of FrostFS.Container operation. @@ -291,35 +292,58 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re return } - res, err := h.search(ctx, bktInfo.CID, key, val, object.MatchStringEqual) + objID, err := h.findObjectByAttribute(ctx, log, bktInfo.CID, key, val) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) - response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) + if errors.Is(err, io.EOF) { + response.Error(c, err.Error(), fasthttp.StatusNotFound) + return + } + + response.Error(c, err.Error(), fasthttp.StatusBadRequest) return } + var addrObj oid.Address + addrObj.SetContainer(bktInfo.CID) + addrObj.SetObject(objID) + + f(ctx, *h.newRequest(c, log), addrObj) +} + +func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { + res, err := h.search(ctx, cnrID, attrKey, attrVal, object.MatchStringEqual) + if err != nil { + log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + return oid.ID{}, fmt.Errorf("could not search for objects: %w", err) + } defer res.Close() buf := make([]oid.ID, 1) n, err := res.Read(buf) if n == 0 { - if errors.Is(err, io.EOF) { + switch { + case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): + log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName) + return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal) + case errors.Is(err, io.EOF): log.Error(logs.ObjectNotFound, zap.Error(err)) - response.Error(c, "object not found", fasthttp.StatusNotFound) - return + return oid.ID{}, fmt.Errorf("object not found: %w", err) + default: + log.Error(logs.ReadObjectListFailed, zap.Error(err)) + return oid.ID{}, fmt.Errorf("read object list failed: %w", err) } - - log.Error(logs.ReadObjectListFailed, zap.Error(err)) - response.Error(c, "read object list failed: "+err.Error(), fasthttp.StatusBadRequest) - return } - var addrObj oid.Address - addrObj.SetContainer(bktInfo.CID) - addrObj.SetObject(buf[0]) + return buf[0], nil +} - f(ctx, *h.newRequest(c, log), addrObj) +func (h *Handler) needSearchByFileName(key, val string) bool { + if key != attrFilePath || !h.config.EnableFilepathFallback() { + return false + } + + return strings.HasPrefix(val, "/") && strings.Count(val, "/") == 1 || !strings.Contains(val, "/") } // resolveContainer decode container id, if it's not a valid container id diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 34668a5..86364d9 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -44,6 +44,7 @@ func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, [ } type configMock struct { + additionalSearch bool } func (c *configMock) DefaultTimestamp() bool { @@ -78,6 +79,10 @@ func (c *configMock) NamespaceHeader() string { return "" } +func (c *configMock) EnableFilepathFallback() bool { + return c.additionalSearch +} + type handlerContext struct { key *keys.PrivateKey owner user.ID @@ -199,10 +204,8 @@ func TestBasic(t *testing.T) { require.NoError(t, err) obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] - attr := object.NewAttribute() - attr.SetKey(object.AttributeFilePath) - attr.SetValue(objFileName) - obj.SetAttributes(append(obj.Attributes(), *attr)...) + attr := prepareObjectAttributes(object.AttributeFilePath, objFileName) + obj.SetAttributes(append(obj.Attributes(), attr)...) t.Run("get", func(t *testing.T) { r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID) @@ -251,6 +254,159 @@ func TestBasic(t *testing.T) { }) } +func TestFindObjectByAttribute(t *testing.T) { + hc, err := prepareHandlerContext() + require.NoError(t, err) + hc.cfg.additionalSearch = true + + bktName := "bucket" + cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended) + require.NoError(t, err) + hc.frostfs.SetContainer(cnrID, cnr) + + ctx := context.Background() + ctx = middleware.SetNamespace(ctx, "") + + content := "hello" + r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content) + require.NoError(t, err) + + hc.Handler().Upload(r) + require.Equal(t, r.Response.StatusCode(), http.StatusOK) + + var putRes putResponse + err = json.Unmarshal(r.Response.Body(), &putRes) + require.NoError(t, err) + + testAttrVal1 := "test-attr-val1" + testAttrVal2 := "test-attr-val2" + testAttrVal3 := "test-attr-val3" + + for _, tc := range []struct { + name string + firstAttr object.Attribute + secondAttr object.Attribute + reqAttrKey string + reqAttrValue string + err string + additionalSearch bool + }{ + { + name: "success search by FileName", + firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), + secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), + reqAttrKey: attrFileName, + reqAttrValue: testAttrVal2, + additionalSearch: false, + }, + { + name: "failed search by FileName", + firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), + secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), + reqAttrKey: attrFileName, + reqAttrValue: testAttrVal3, + err: "not found", + additionalSearch: false, + }, + { + name: "success search by FilePath (with additional search)", + firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), + secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), + reqAttrKey: attrFilePath, + reqAttrValue: testAttrVal2, + additionalSearch: true, + }, + { + name: "failed by FilePath (with additional search)", + firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), + secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), + reqAttrKey: attrFilePath, + reqAttrValue: testAttrVal3, + err: "not found", + additionalSearch: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] + obj.SetAttributes(tc.firstAttr, tc.secondAttr) + hc.cfg.additionalSearch = tc.additionalSearch + + objID, err := hc.Handler().findObjectByAttribute(ctx, hc.Handler().log, cnrID, tc.reqAttrKey, tc.reqAttrValue) + if tc.err != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tc.err) + return + } + + require.NoError(t, err) + require.Equal(t, putRes.ObjectID, objID.EncodeToString()) + }) + } +} + +func TestNeedSearchByFileName(t *testing.T) { + hc, err := prepareHandlerContext() + require.NoError(t, err) + + for _, tc := range []struct { + name string + attrKey string + attrVal string + additionalSearch bool + expected bool + }{ + { + name: "need search - not contains slash", + attrKey: attrFilePath, + attrVal: "cat.png", + additionalSearch: true, + expected: true, + }, + { + name: "need search - single lead slash", + attrKey: attrFilePath, + attrVal: "/cat.png", + additionalSearch: true, + expected: true, + }, + { + name: "don't need search - single slash but not lead", + attrKey: attrFilePath, + attrVal: "cats/cat.png", + additionalSearch: true, + expected: false, + }, + { + name: "don't need search - more one slash", + attrKey: attrFilePath, + attrVal: "/cats/cat.png", + additionalSearch: true, + expected: false, + }, + { + name: "don't need search - incorrect attribute key", + attrKey: attrFileName, + attrVal: "cat.png", + additionalSearch: true, + expected: false, + }, + { + name: "don't need search - additional search disabled", + attrKey: attrFilePath, + attrVal: "cat.png", + additionalSearch: false, + expected: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + hc.cfg.additionalSearch = tc.additionalSearch + + res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal) + require.Equal(t, tc.expected, res) + }) + } +} + func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) @@ -283,6 +439,13 @@ func prepareGetZipped(ctx context.Context, bucket, prefix string) *fasthttp.Requ return r } +func prepareObjectAttributes(attrKey, attrValue string) object.Attribute { + attr := object.NewAttribute() + attr.SetKey(attrKey) + attr.SetValue(attrValue) + return *attr +} + const ( keyAttr = "User-Attribute" valAttr = "user value" diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 4dfa21f..8c8b336 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -87,4 +87,5 @@ const ( MultinetDialFail = "multinet dial failed" FailedToLoadMultinetConfig = "failed to load multinet config" MultinetConfigWontBeUpdated = "multinet config won't be updated" + ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" ) From 1be92fa4befc8f95bf94f5aff39e05c87c57d7f8 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Mon, 2 Dec 2024 11:45:30 +0300 Subject: [PATCH 492/548] [#166] Fix getting s3 object with the FrostFS OID name Prioritize getting s3 object with the key, which equals to valid FrostFS OID, rather than getting non-existent object with OID via native protocol for GET and HEAD requests Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 4 +- internal/data/{bucket.go => info.go} | 0 internal/{api => data}/tree.go | 17 ++- internal/handler/browse.go | 2 +- internal/handler/download.go | 49 +++++++-- internal/handler/handler.go | 104 +++++------------- internal/handler/handler_test.go | 33 ++++-- internal/handler/head.go | 34 +++++- internal/handler/utils.go | 11 +- internal/{api => }/layer/tree_service.go | 6 +- internal/logs/logs.go | 5 + .../{pool_wrapper.go => tree_pool_wrapper.go} | 0 tree/tree.go | 52 ++++++--- 13 files changed, 178 insertions(+), 139 deletions(-) rename internal/data/{bucket.go => info.go} (100%) rename internal/{api => data}/tree.go (61%) rename internal/{api => }/layer/tree_service.go (64%) rename internal/service/frostfs/{pool_wrapper.go => tree_pool_wrapper.go} (100%) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 9adaec2..23a752a 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -508,10 +508,10 @@ func (a *app) Serve() { close(a.webDone) }() - handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) + handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) // Configure router. - a.configureRouter(handler) + a.configureRouter(handle) a.startServices() a.initServers(a.ctx) diff --git a/internal/data/bucket.go b/internal/data/info.go similarity index 100% rename from internal/data/bucket.go rename to internal/data/info.go diff --git a/internal/api/tree.go b/internal/data/tree.go similarity index 61% rename from internal/api/tree.go rename to internal/data/tree.go index 5b1d608..fcf8add 100644 --- a/internal/api/tree.go +++ b/internal/data/tree.go @@ -1,4 +1,4 @@ -package api +package data import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -7,12 +7,21 @@ import ( // NodeVersion represent node from tree service. type NodeVersion struct { BaseNodeVersion - DeleteMarker bool - IsPrefixNode bool } // BaseNodeVersion is minimal node info from tree service. // Basically used for "system" object. type BaseNodeVersion struct { - OID oid.ID + ID uint64 + OID oid.ID + IsDeleteMarker bool +} + +type NodeInfo struct { + Meta []NodeMeta +} + +type NodeMeta interface { + GetKey() string + GetValue() []byte } diff --git a/internal/handler/browse.go b/internal/handler/browse.go index c54ab76..2938b5c 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -170,7 +170,7 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn objects: make([]ResponseObject, 0, len(nodes)), } for _, node := range nodes { - meta := node.GetMeta() + meta := node.Meta if meta == nil { continue } diff --git a/internal/handler/download.go b/internal/handler/download.go index cd4e55a..de27fa3 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -4,12 +4,14 @@ import ( "archive/zip" "bufio" "context" + "errors" "fmt" "io" "net/http" "net/url" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" @@ -23,21 +25,46 @@ import ( // DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { - oidURLParam := c.UserValue("oid").(string) - downloadQueryParam := c.QueryArgs().GetBool("download") + cidParam := c.UserValue("cid").(string) + oidParam := c.UserValue("oid").(string) + downloadParam := c.QueryArgs().GetBool("download") - switch { - case isObjectID(oidURLParam): - h.byNativeAddress(c, h.receiveFile) - case !isContainerRoot(oidURLParam) && (downloadQueryParam || !isDir(oidURLParam)): - h.byS3Path(c, h.receiveFile) - default: - h.browseIndex(c) + ctx := utils.GetContextFromRequest(c) + log := utils.GetReqLogOrDefault(ctx, h.log).With( + zap.String("cid", cidParam), + zap.String("oid", oidParam), + ) + + bktInfo, err := h.getBucketInfo(ctx, cidParam, log) + if err != nil { + logAndSendBucketError(c, log, err) + return + } + + checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) + if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + logAndSendBucketError(c, log, checkS3Err) + return + } + + req := h.newRequest(c, log) + + var objID oid.ID + if checkS3Err == nil && shouldDownload(oidParam, downloadParam) { + h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile) + } else if err = objID.DecodeString(oidParam); err == nil { + h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile) + } else { + h.browseIndex(c, checkS3Err != nil) } } -func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { - return &request{ +func shouldDownload(oidParam string, downloadParam bool) bool { + return !isDir(oidParam) || downloadParam +} + +func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) request { + return request{ RequestCtx: ctx, log: log, } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 1839bf0..3805c2d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -11,9 +11,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -165,7 +165,7 @@ type Handler struct { ownerID *user.ID config Config containerResolver ContainerResolver - tree *tree.Tree + tree layer.TreeService cache *cache.BucketCache workerPool *ants.Pool } @@ -178,7 +178,7 @@ type AppParams struct { Cache *cache.BucketCache } -func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { +func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler { return &Handler{ log: params.Logger, frostfs: params.FrostFS, @@ -193,77 +193,34 @@ func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Poo // byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byNativeAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { - idCnr, _ := c.UserValue("cid").(string) - idObj, _ := url.PathUnescape(c.UserValue("oid").(string)) - - ctx := utils.GetContextFromRequest(c) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With(zap.String("cid", idCnr), zap.String("oid", idObj)) - - bktInfo, err := h.getBucketInfo(ctx, idCnr, log) - if err != nil { - logAndSendBucketError(c, log, err) - return - } - - objID := new(oid.ID) - if err = objID.DecodeString(idObj); err != nil { - log.Error(logs.WrongObjectID, zap.Error(err)) - response.Error(c, "wrong object id", fasthttp.StatusBadRequest) - return - } - - addr := newAddress(bktInfo.CID, *objID) - - f(ctx, *h.newRequest(c, log), addr) +func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID, objID oid.ID, handler func(context.Context, request, oid.Address)) { + addr := newAddress(cnrID, objID) + handler(ctx, req, addr) } // byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that // resolves object address from S3-like path /. -func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { - bucketname := c.UserValue("cid").(string) - key := c.UserValue("oid").(string) +func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path string, handler func(context.Context, request, oid.Address)) { + c, log := req.RequestCtx, req.log - ctx := utils.GetContextFromRequest(c) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With(zap.String("bucketname", bucketname), zap.String("key", key)) - - unescapedKey, err := url.QueryUnescape(key) + foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) if err != nil { logAndSendBucketError(c, log, err) return } - - bktInfo, err := h.getBucketInfo(ctx, bucketname, log) - if err != nil { - logAndSendBucketError(c, log, err) - return - } - - foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey) - if err != nil { - if errors.Is(err, tree.ErrNodeAccessDenied) { - response.Error(c, "Access Denied", fasthttp.StatusForbidden) - } else { - response.Error(c, "object wasn't found", fasthttp.StatusNotFound) - log.Error(logs.GetLatestObjectVersion, zap.Error(err)) - } - return - } - if foundOid.DeleteMarker { + if foundOID.IsDeleteMarker { log.Error(logs.ObjectWasDeleted) response.Error(c, "object deleted", fasthttp.StatusNotFound) return } - addr := newAddress(bktInfo.CID, foundOid.OID) - f(ctx, *h.newRequest(c, log), addr) + addr := newAddress(cnrID, foundOID.OID) + handler(ctx, h.newRequest(c, log), addr) } // byAttribute is a wrapper similar to byNativeAddress. -func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { - scid, _ := c.UserValue("cid").(string) +func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Context, request, oid.Address)) { + cidParam, _ := c.UserValue("cid").(string) key, _ := c.UserValue("attr_key").(string) val, _ := c.UserValue("attr_val").(string) @@ -272,21 +229,21 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re key, err := url.QueryUnescape(key) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_key", key), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), zap.Error(err)) response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) return } val, err = url.QueryUnescape(val) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_val", val), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), zap.Error(err)) response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) return } - log = log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) + log = log.With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val)) - bktInfo, err := h.getBucketInfo(ctx, scid, log) + bktInfo, err := h.getBucketInfo(ctx, cidParam, log) if err != nil { logAndSendBucketError(c, log, err) return @@ -303,11 +260,11 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re return } - var addrObj oid.Address - addrObj.SetContainer(bktInfo.CID) - addrObj.SetObject(objID) + var addr oid.Address + addr.SetContainer(bktInfo.CID) + addr.SetObject(objID) - f(ctx, *h.newRequest(c, log), addrObj) + handler(ctx, h.newRequest(c, log), addr) } func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { @@ -412,7 +369,7 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } -func (h *Handler) browseIndex(c *fasthttp.RequestCtx) { +func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) { if !h.config.IndexPageEnabled() { c.SetStatusCode(fasthttp.StatusNotFound) return @@ -438,18 +395,9 @@ func (h *Handler) browseIndex(c *fasthttp.RequestCtx) { } listFunc := h.getDirObjectsS3 - isNativeList := false - - err = h.tree.CheckSettingsNodeExist(ctx, bktInfo) - if err != nil { - if errors.Is(err, tree.ErrNodeNotFound) { - // tree probe failed, try to use native - listFunc = h.getDirObjectsNative - isNativeList = true - } else { - logAndSendBucketError(c, log, err) - return - } + if isNativeList { + // tree probe failed, trying to use native + listFunc = h.getDirObjectsNative } h.browseObjects(c, browseParams{ diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 86364d9..14f9c98 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -14,8 +14,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" @@ -32,14 +32,29 @@ import ( "go.uber.org/zap" ) -type treeClientMock struct { +type treeServiceMock struct { + system map[string]map[string]*data.BaseNodeVersion } -func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) { - return nil, nil +func newTreeService() *treeServiceMock { + return &treeServiceMock{ + system: make(map[string]map[string]*data.BaseNodeVersion), + } } -func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) { +func (t *treeServiceMock) CheckSettingsNodeExists(context.Context, *data.BucketInfo) error { + _, ok := t.system["bucket-settings"] + if !ok { + return layer.ErrNodeNotFound + } + return nil +} + +func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeInfo, string, error) { + return nil, "", nil +} + +func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*data.NodeVersion, error) { return nil, nil } @@ -89,7 +104,7 @@ type handlerContext struct { h *Handler frostfs *TestFrostFS - tree *treeClientMock + tree *treeServiceMock cfg *configMock } @@ -130,14 +145,14 @@ func prepareHandlerContext() (*handlerContext, error) { }), } - treeMock := &treeClientMock{} + treeMock := newTreeService() cfgMock := &configMock{} - workerPool, err := ants.NewPool(1000) + workerPool, err := ants.NewPool(1) if err != nil { return nil, err } - handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool) + handler := New(params, cfgMock, treeMock, workerPool) return &handlerContext{ key: key, diff --git a/internal/handler/head.go b/internal/handler/head.go index ccd6a91..f2e9f38 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -2,11 +2,13 @@ package handler import ( "context" + "errors" "io" "net/http" "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -102,14 +104,36 @@ func idsToResponse(resp *fasthttp.Response, obj *object.Object) { // HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format. func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { - test, _ := c.UserValue("oid").(string) - var id oid.ID + cidParam, _ := c.UserValue("cid").(string) + oidParam, _ := c.UserValue("oid").(string) - err := id.DecodeString(test) + ctx := utils.GetContextFromRequest(c) + log := utils.GetReqLogOrDefault(ctx, h.log).With( + zap.String("cid", cidParam), + zap.String("oid", oidParam), + ) + + bktInfo, err := h.getBucketInfo(ctx, cidParam, log) if err != nil { - h.byS3Path(c, h.headObject) + logAndSendBucketError(c, log, err) + return + } + checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) + if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + logAndSendBucketError(c, log, checkS3Err) + return + } + + req := h.newRequest(c, log) + + var objID oid.ID + if checkS3Err == nil { + h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject) + } else if err = objID.DecodeString(oidParam); err == nil { + h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) } else { - h.byNativeAddress(c, h.headObject) + logAndSendBucketError(c, log, checkS3Err) + return } } diff --git a/internal/handler/utils.go b/internal/handler/utils.go index b537d64..d09ed23 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -42,16 +42,7 @@ func bearerToken(ctx context.Context) *bearer.Token { } func isDir(name string) bool { - return strings.HasSuffix(name, "/") -} - -func isObjectID(s string) bool { - var objID oid.ID - return objID.DecodeString(s) == nil -} - -func isContainerRoot(key string) bool { - return key == "" + return name == "" || strings.HasSuffix(name, "/") } func loadAttributes(attrs []object.Attribute) map[string]string { diff --git a/internal/api/layer/tree_service.go b/internal/layer/tree_service.go similarity index 64% rename from internal/api/layer/tree_service.go rename to internal/layer/tree_service.go index beb1e7a..ff80543 100644 --- a/internal/api/layer/tree_service.go +++ b/internal/layer/tree_service.go @@ -4,13 +4,15 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) // TreeService provide interface to interact with tree service using s3 data models. type TreeService interface { - GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) + GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) + GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) + CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error } var ( diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 8c8b336..7a04064 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -79,6 +79,11 @@ const ( InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go FailedToUnescapeQuery = "failed to unescape query" + FailedToParseAddressInTreeNode = "failed to parse object addr in tree node" + SettingsNodeInvalidOwnerKey = "settings node: invalid owner key" + SystemNodeHasMultipleIDs = "system node has multiple ids" + FailedToRemoveOldSystemNode = "failed to remove old system node" + BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids" ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" diff --git a/internal/service/frostfs/pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go similarity index 100% rename from internal/service/frostfs/pool_wrapper.go rename to internal/service/frostfs/tree_pool_wrapper.go diff --git a/tree/tree.go b/tree/tree.go index 40209a5..bf0aff9 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -6,9 +6,8 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -118,7 +117,7 @@ func (n *treeNode) FileName() (string, bool) { return value, ok } -func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { +func newNodeVersion(node NodeResponse) (*data.NodeVersion, error) { tNode, err := newTreeNode(node) if err != nil { return nil, fmt.Errorf("invalid tree node: %w", err) @@ -127,20 +126,30 @@ func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { return newNodeVersionFromTreeNode(tNode), nil } -func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { +func newNodeVersionFromTreeNode(treeNode *treeNode) *data.NodeVersion { _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) - size, _ := treeNode.Get(sizeKV) - version := &api.NodeVersion{ - BaseNodeVersion: api.BaseNodeVersion{ - OID: treeNode.ObjID, + version := &data.NodeVersion{ + BaseNodeVersion: data.BaseNodeVersion{ + OID: treeNode.ObjID, + IsDeleteMarker: isDeleteMarker, }, - DeleteMarker: isDeleteMarker, - IsPrefixNode: size == "", } return version } +func newNodeInfo(node NodeResponse) data.NodeInfo { + nodeMeta := node.GetMeta() + nodeInfo := data.NodeInfo{ + Meta: make([]data.NodeMeta, 0, len(nodeMeta)), + } + for _, meta := range nodeMeta { + nodeInfo.Meta = append(nodeInfo.Meta, meta) + } + + return nodeInfo +} + func newMultiNode(nodes []NodeResponse) (*multiSystemNode, error) { var ( err error @@ -180,7 +189,7 @@ func (m *multiSystemNode) Old() []*treeNode { return m.nodes[1:] } -func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { +func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) { nodes, err := c.GetVersions(ctx, cnrID, objectName) if err != nil { return nil, err @@ -210,7 +219,7 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string return c.service.GetNodes(ctx, p) } -func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error { +func (c *Tree) CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error { _, err := c.getSystemNode(ctx, bktInfo, settingsFileName) if err != nil { return err @@ -236,7 +245,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name nodes = filterMultipartNodes(nodes) if len(nodes) == 0 { - return nil, ErrNodeNotFound + return nil, layer.ErrNodeNotFound } return newMultiNode(nodes) @@ -298,14 +307,14 @@ func pathFromName(objectName string) []string { return strings.Split(objectName, separator) } -func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]NodeResponse, string, error) { +func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) { rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix) if err != nil { return nil, "", err } subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false) if err != nil { - if errors.Is(err, layer.ErrNodeNotFound) { + if errors.Is(err, ErrNodeNotFound) { return nil, "", nil } return nil, "", err @@ -340,14 +349,23 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, nodesMap[fileName] = nodes } - result := make([]NodeResponse, 0, len(subTree)) + result := make([]data.NodeInfo, 0, len(subTree)) for _, nodes := range nodesMap { - result = append(result, nodes...) + result = append(result, nodeResponseToNodeInfo(nodes)...) } return result, strings.TrimSuffix(prefix, tailPrefix), nil } +func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo { + nodesInfo := make([]data.NodeInfo, 0, len(nodes)) + for _, node := range nodes { + nodesInfo = append(nodesInfo, newNodeInfo(node)) + } + + return nodesInfo +} + func (c *Tree) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) ([]uint64, string, error) { rootID := []uint64{0} path := strings.Split(prefix, separator) From a945a947ac7b7833fb97fa81478c550e126f2783 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 16 Dec 2024 15:56:27 +0300 Subject: [PATCH 493/548] [#183] Unlink API.md to README file This is useful for auto-generated document tools which parse docs dir. Signed-off-by: Alex Vanin --- README.md | 139 +---------------------------------------- docs/api.md | 4 +- docs/authentication.md | 108 ++++++++++++++++++++++++++++++++ docs/nns.md | 36 +++++++++++ 4 files changed, 148 insertions(+), 139 deletions(-) create mode 100644 docs/authentication.md create mode 100644 docs/nns.md diff --git a/README.md b/README.md index e1af0eb..adf793c 100644 --- a/README.md +++ b/README.md @@ -217,41 +217,8 @@ Also, in case of downloading, you need to have a file inside a container. ### NNS In all download/upload routes you can use container name instead of its id (`$CID`). +Read more about it in [docs/nns.md](./docs/nns.md). -Steps to start using name resolving: - -1. Enable NNS resolving in config (`rpc_endpoint` must be a valid neo rpc node, see [configs](./config) for other examples): - -```yaml -rpc_endpoint: http://morph-chain.frostfs.devenv:30333 -resolve_order: - - nns -``` - -2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env) -you can check if your container (e.g. with `container-name` name) is registered in NNS: - -```shell -$ curl -s --data '{"id":1,"jsonrpc":"2.0","method":"getcontractstate","params":[1]}' \ - http://morph-chain.frostfs.devenv:30333 | jq -r '.result.hash' - -0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 - -$ docker exec -it morph_chain neo-go \ - contract testinvokefunction \ - -r http://morph-chain.frostfs.devenv:30333 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 \ - resolve string:container-name.container int:16 \ - | jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \ - | base64 -d && echo - -7f3vvkw4iTiS5ZZbu5BQXEmJtETWbi3uUjLNaSs29xrL -``` - -3. Use container name instead of its `$CID`. For example: - -```shell -$ curl http://localhost:8082/get_by_attribute/container-name/FileName/object-name -``` #### Create a container @@ -462,109 +429,7 @@ object ID, like this: #### Authentication -You can always upload files to public containers (open for anyone to put -objects into), but for restricted containers you need to explicitly allow PUT -operations for a request signed with your HTTP Gateway keys. - -If you don't want to manage gateway's secret keys and adjust policies when -gateway configuration changes (new gate, key rotation, etc) or you plan to use -public services, there is an option to let your application backend (or you) to -issue Bearer Tokens and pass them from the client via gate down to FrostFS level -to grant access. - -FrostFS Bearer Token basically is a container owner-signed policy (refer to FrostFS -documentation for more details). There are two options to pass them to gateway: - * "Authorization" header with "Bearer" type and base64-encoded token in - credentials field - * "Bearer" cookie with base64-encoded token contents - -For example, you have a mobile application frontend with a backend part storing -data in FrostFS. When a user authorizes in the mobile app, the backend issues a FrostFS -Bearer token and provides it to the frontend. Then, the mobile app may generate -some data and upload it via any available FrostFS HTTP Gateway by adding -the corresponding header to the upload request. Accessing policy protected data -works the same way. - -##### Example -In order to generate a bearer token, you need to have wallet (which will be used to sign the token) - -1. Suppose you have a container with private policy for wallet key - -``` -$ frostfs-cli container create -r --wallet -policy --basic-acl 0 --await -CID: 9dfzyvq82JnFqp5svxcREf2iy6XNuifYcJPusEDnGK9Z - -$ frostfs-cli ape-manager add -r --wallet \ - --target-type container --target-name 9dfzyvq82JnFqp5svxcREf2iy6XNuifYcJPusEDnGK9Z \ - --rule "allow Object.* RequestCondition:"\$Actor:publicKey"=03b09baabff3f6107c7e9acb8721a6fc5618d45b50247a314d82e548702cce8cd5 *" \ - --chain-id -``` - - -2. Form a Bearer token (10000 is lifetime expiration in epoch) to impersonate - HTTP Gateway request as wallet signed request and save it to **bearer.json**: -``` -{ - "body": { - "allowImpersonate": true, - "lifetime": { - "exp": "10000", - "nbf": "0", - "iat": "0" - } - }, - "signature": null -} -``` - -3. Sign it with the wallet: -``` -$ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w -``` - -4. Encode to base64 to use in header: -``` -$ base64 -w 0 signed.json -# output: Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw== -``` - -After that, the Bearer token can be used: - -``` -$ curl -F 'file=@cat.jpeg;filename=cat.jpeg' -H "Authorization: Bearer Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw==" \ - http://localhost:8082/upload/BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K -# output: -# { -# "object_id": "DhfES9nVrFksxGDD2jQLunGADfrXExxNwqXbDafyBn9X", -# "container_id": "BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K" -# } -``` - -##### Note: Bearer Token owner - -You can specify exact key who can use Bearer Token (gateway wallet address). -To do this, encode wallet address in base64 format - -``` -$ echo 'NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3' | base58 --decode | base64 -# output: NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg== -``` - -Then specify this value in Bearer Token Json -``` -{ - "body": { - "ownerID": { - "value": "NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg==" - }, - ... -``` - -##### Note: Policy override - -Instead of impersonation, you can define the set of policies that will be applied -to the request sender. This allows to restrict access to specific operation and -specific objects without giving full impersonation control to the token user. +Read more about request authentication in [docs/authentication.md](./docs/authemtnication.md) ### Metrics and Pprof diff --git a/docs/api.md b/docs/api.md index f7eb3a4..e59956a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -8,7 +8,7 @@ | `/zip/{cid}/{prefix}` | [Download objects in archive](#download-zip) | **Note:** `cid` parameter can be base58 encoded container ID or container name -(the name must be registered in NNS, see appropriate section in [README](../README.md#nns)). +(the name must be registered in NNS, see appropriate section in [nns.md](./nns.md)). Route parameters can be: @@ -18,7 +18,7 @@ Route parameters can be: ### Bearer token -All routes can accept [bearer token](../README.md#authentication) from: +All routes can accept [bearer token](./authentication.md) from: * `Authorization` header with `Bearer` type and base64-encoded token in credentials field diff --git a/docs/authentication.md b/docs/authentication.md new file mode 100644 index 0000000..d8bb235 --- /dev/null +++ b/docs/authentication.md @@ -0,0 +1,108 @@ +# Request authentication + +HTTP Gateway does not authorize requests. Gateway converts HTTP request to a +FrostFS request and signs it with its own private key. + +You can always upload files to public containers (open for anyone to put +objects into), but for restricted containers you need to explicitly allow PUT +operations for a request signed with your HTTP Gateway keys. + +If you don't want to manage gateway's secret keys and adjust policies when +gateway configuration changes (new gate, key rotation, etc) or you plan to use +public services, there is an option to let your application backend (or you) to +issue Bearer Tokens and pass them from the client via gate down to FrostFS level +to grant access. + +FrostFS Bearer Token basically is a container owner-signed policy (refer to FrostFS +documentation for more details). There are two options to pass them to gateway: +* "Authorization" header with "Bearer" type and base64-encoded token in + credentials field +* "Bearer" cookie with base64-encoded token contents + +For example, you have a mobile application frontend with a backend part storing +data in FrostFS. When a user authorizes in the mobile app, the backend issues a FrostFS +Bearer token and provides it to the frontend. Then, the mobile app may generate +some data and upload it via any available FrostFS HTTP Gateway by adding +the corresponding header to the upload request. Accessing policy protected data +works the same way. + +##### Example +In order to generate a bearer token, you need to have wallet (which will be used to sign the token) + +1. Suppose you have a container with private policy for wallet key + +``` +$ frostfs-cli container create -r --wallet -policy --basic-acl 0 --await +CID: 9dfzyvq82JnFqp5svxcREf2iy6XNuifYcJPusEDnGK9Z + +$ frostfs-cli ape-manager add -r --wallet \ + --target-type container --target-name 9dfzyvq82JnFqp5svxcREf2iy6XNuifYcJPusEDnGK9Z \ + --rule "allow Object.* RequestCondition:"\$Actor:publicKey"=03b09baabff3f6107c7e9acb8721a6fc5618d45b50247a314d82e548702cce8cd5 *" \ + --chain-id +``` + + +2. Form a Bearer token (10000 is lifetime expiration in epoch) to impersonate + HTTP Gateway request as wallet signed request and save it to **bearer.json**: +``` +{ + "body": { + "allowImpersonate": true, + "lifetime": { + "exp": "10000", + "nbf": "0", + "iat": "0" + } + }, + "signature": null +} +``` + +3. Sign it with the wallet: +``` +$ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w +``` + +4. Encode to base64 to use in header: +``` +$ base64 -w 0 signed.json +# output: Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw== +``` + +After that, the Bearer token can be used: + +``` +$ curl -F 'file=@cat.jpeg;filename=cat.jpeg' -H "Authorization: Bearer Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw==" \ + http://localhost:8082/upload/BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K +# output: +# { +# "object_id": "DhfES9nVrFksxGDD2jQLunGADfrXExxNwqXbDafyBn9X", +# "container_id": "BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K" +# } +``` + +##### Note: Bearer Token owner + +You can specify exact key who can use Bearer Token (gateway wallet address). +To do this, encode wallet address in base64 format + +``` +$ echo 'NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3' | base58 --decode | base64 +# output: NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg== +``` + +Then specify this value in Bearer Token Json +``` +{ + "body": { + "ownerID": { + "value": "NezFK4ujidF+X7bB88uzREQzRQeAvdj3Gg==" + }, + ... +``` + +##### Note: Policy override + +Instead of impersonation, you can define the set of policies that will be applied +to the request sender. This allows to restrict access to specific operation and +specific objects without giving full impersonation control to the token user. diff --git a/docs/nns.md b/docs/nns.md new file mode 100644 index 0000000..acb9f21 --- /dev/null +++ b/docs/nns.md @@ -0,0 +1,36 @@ +# Nicename Resolving with NNS + +Steps to start using name resolving: + +1. Enable NNS resolving in config (`rpc_endpoint` must be a valid neo rpc node, see [configs](./config) for other examples): + +```yaml +rpc_endpoint: http://morph-chain.frostfs.devenv:30333 +resolve_order: + - nns +``` + +2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env) + you can check if your container (e.g. with `container-name` name) is registered in NNS: + +```shell +$ curl -s --data '{"id":1,"jsonrpc":"2.0","method":"getcontractstate","params":[1]}' \ + http://morph-chain.frostfs.devenv:30333 | jq -r '.result.hash' + +0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 + +$ docker exec -it morph_chain neo-go \ + contract testinvokefunction \ + -r http://morph-chain.frostfs.devenv:30333 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 \ + resolve string:container-name.container int:16 \ + | jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \ + | base64 -d && echo + +7f3vvkw4iTiS5ZZbu5BQXEmJtETWbi3uUjLNaSs29xrL +``` + +3. Use container name instead of its `$CID`. For example: + +```shell +$ curl http://localhost:8082/get_by_attribute/container-name/FileName/object-name +``` From a658f3adc0ec2bf7f1dbb13701360eeb5c4a4470 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Mon, 16 Dec 2024 13:01:50 +0300 Subject: [PATCH 494/548] [#181] index_page: Ignore deleted objects in versioned buckets Signed-off-by: Nikita Zinkevich --- internal/handler/browse.go | 42 ++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/internal/handler/browse.go b/internal/handler/browse.go index 2938b5c..64ad1f5 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -22,12 +22,13 @@ import ( ) const ( - dateFormat = "02-01-2006 15:04" - attrOID = "OID" - attrCreated = "Created" - attrFileName = "FileName" - attrFilePath = "FilePath" - attrSize = "Size" + dateFormat = "02-01-2006 15:04" + attrOID = "OID" + attrCreated = "Created" + attrFileName = "FileName" + attrFilePath = "FilePath" + attrSize = "Size" + attrDeleteMarker = "IsDeleteMarker" ) type ( @@ -39,23 +40,25 @@ type ( Objects []ResponseObject } ResponseObject struct { - OID string - Created string - FileName string - FilePath string - Size string - IsDir bool - GetURL string + OID string + Created string + FileName string + FilePath string + Size string + IsDir bool + GetURL string + IsDeleteMarker bool } ) func newListObjectsResponseS3(attrs map[string]string) ResponseObject { return ResponseObject{ - Created: formatTimestamp(attrs[attrCreated]), - OID: attrs[attrOID], - FileName: attrs[attrFileName], - Size: attrs[attrSize], - IsDir: attrs[attrOID] == "", + Created: formatTimestamp(attrs[attrCreated]), + OID: attrs[attrOID], + FileName: attrs[attrFileName], + Size: attrs[attrSize], + IsDir: attrs[attrOID] == "", + IsDeleteMarker: attrs[attrDeleteMarker] == "true", } } @@ -179,6 +182,9 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn attrs[m.GetKey()] = string(m.GetValue()) } obj := newListObjectsResponseS3(attrs) + if obj.IsDeleteMarker { + continue + } obj.FilePath = prefix + obj.FileName obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath) result.objects = append(result.objects, obj) From d32ac4b5370f93ea488cfa2191f1ed15e35800ba Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 13:49:13 +0300 Subject: [PATCH 495/548] Release v0.32.0 Signed-off-by: Alex Vanin --- CHANGELOG.md | 16 +++++++++++++++- VERSION | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc422d6..e528b8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ This document outlines major changes between releases. ## [Unreleased] +## [0.32.0] - Khumbu - 2024-12-20 + +### Fixed +- Getting S3 object with FrostFS Object ID-like key (#166) +- Ignore delete marked objects in versioned bucket in index page (#181) + +### Added +- Metric of dropped logs by log sampler (#150) +- Fallback FileName attribute search during FilePath attribute search (#174) + +### Changed +- Updated tree service pool without api-go dependency (#178) + ## [0.31.0] - Rongbuk - 2024-11-20 ### Fixed @@ -170,4 +183,5 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.30.2]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.1...v0.30.2 [0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.2...v0.30.3 [0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.3...v0.31.0 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...master \ No newline at end of file +[0.32.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...v0.32.0 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.0...master \ No newline at end of file diff --git a/VERSION b/VERSION index 7021025..420000f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.31.0 +v0.32.0 From a4e3767d4bb2c6853bd52e1eabffb1729d5dd1fd Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Wed, 18 Dec 2024 14:31:12 +0300 Subject: [PATCH 496/548] [#175] Adopt 1.6.* aio versoins in integration tests Signed-off-by: Roman Loginov --- cmd/http-gw/integration_test.go | 45 +++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index dd4d60f..6a4d428 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -14,6 +14,7 @@ import ( "net/http" "os" "sort" + "strings" "testing" "time" @@ -54,6 +55,7 @@ func TestIntegration(t *testing.T) { "1.2.7", "1.3.0", "1.5.0", + "1.6.5", } key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) @@ -70,21 +72,26 @@ func TestIntegration(t *testing.T) { ctx, cancel2 := context.WithCancel(rootCtx) aioContainer := createDockerContainer(ctx, t, aioImage+version) + if strings.HasPrefix(version, "1.6") { + registerUser(t, ctx, aioContainer, file.Name()) + } + + // See the logs from the command execution. server, cancel := runServer(file.Name()) clientPool := getPool(ctx, t, key) - CID, err := createContainer(ctx, t, clientPool, ownerID, version) + CID, err := createContainer(ctx, t, clientPool, ownerID) require.NoError(t, err, version) token := makeBearerToken(t, key, ownerID, version) - t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID, version) }) + t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) }) t.Run("put with bearer token in header"+version, func(t *testing.T) { putWithBearerTokenInHeader(ctx, t, clientPool, CID, token) }) t.Run("put with bearer token in cookie"+version, func(t *testing.T) { putWithBearerTokenInCookie(ctx, t, clientPool, CID, token) }) t.Run("put with duplicate keys "+version, func(t *testing.T) { putWithDuplicateKeys(t, CID) }) - t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, ownerID, CID, version) }) - t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID, version) }) - t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, ownerID, CID, version) }) - t.Run("test namespaces "+version, func(t *testing.T) { checkNamespaces(ctx, t, clientPool, ownerID, CID, version) }) + t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, ownerID, CID) }) + t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID) }) + t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, ownerID, CID) }) + t.Run("test namespaces "+version, func(t *testing.T) { checkNamespaces(ctx, t, clientPool, ownerID, CID) }) cancel() server.Wait() @@ -107,7 +114,7 @@ func runServer(pathToWallet string) (App, context.CancelFunc) { return application, cancel } -func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID, version string) { +func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID) { url := testHost + "/upload/" + CID.String() makePutRequestAndCheck(ctx, t, p, CID, url) @@ -255,7 +262,7 @@ func putWithDuplicateKeys(t *testing.T, CID cid.ID) { require.Equal(t, http.StatusBadRequest, resp.StatusCode) } -func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { +func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID) { content := "content of file" attributes := map[string]string{ "some-attr": "some-get-value", @@ -302,7 +309,7 @@ func checkGetByAttrResponse(t *testing.T, resp *http.Response, content string, a } } -func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { +func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID) { keyAttr, valAttr := "some-attr", "some-get-by-attr-value" content := "content of file" attributes := map[string]string{keyAttr: valAttr} @@ -324,7 +331,7 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID checkGetByAttrResponse(t, resp, content, expectedAttr) } -func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { +func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID) { names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"} contents := []string{"content of file1", "content of file2"} attributes1 := map[string]string{object.AttributeFilePath: names[0]} @@ -389,7 +396,7 @@ func checkZip(t *testing.T, data []byte, length int64, names, contents []string) } } -func checkNamespaces(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) { +func checkNamespaces(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID) { content := "content of file" attributes := map[string]string{ "some-attr": "some-get-value", @@ -426,7 +433,7 @@ func checkNamespaces(ctx context.Context, t *testing.T, clientPool *pool.Pool, o func createDockerContainer(ctx context.Context, t *testing.T, image string) testcontainers.Container { req := testcontainers.ContainerRequest{ Image: image, - WaitingFor: wait.NewLogStrategy("aio container started").WithStartupTimeout(30 * time.Second), + WaitingFor: wait.NewLogStrategy("aio container started").WithStartupTimeout(2 * time.Minute), Name: "aio", Hostname: "aio", NetworkMode: "host", @@ -466,7 +473,7 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool return clientPool } -func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, version string) (cid.ID, error) { +func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID) (cid.ID, error) { var policy netmap.PlacementPolicy err := policy.DecodeString("REP 1") require.NoError(t, err) @@ -526,6 +533,18 @@ func putObject(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID return id.ObjectID } +func registerUser(t *testing.T, ctx context.Context, aioContainer testcontainers.Container, pathToWallet string) { + err := aioContainer.CopyFileToContainer(ctx, pathToWallet, "/usr/wallet.json", 644) + require.NoError(t, err) + + _, err = aioContainer.Exec(ctx, []string{ + "/usr/bin/frostfs-s3-authmate", "register-user", + "--wallet", "/usr/wallet.json", + "--rpc-endpoint", "http://localhost:30333", + "--contract-wallet", "/config/s3-gw-wallet.json"}) + require.NoError(t, err) +} + func makeBearerToken(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string) string { tkn := new(bearer.Token) tkn.ForUser(ownerID) From 9551f34f00ffa1367ebe703609647b42ba3b1aa8 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Fri, 29 Nov 2024 09:48:44 +0300 Subject: [PATCH 497/548] [#163] Support JSON bearer token Signed-off-by: Roman Loginov --- cmd/http-gw/integration_test.go | 22 ++++++++++----- tokens/bearer-token.go | 18 +++++++++---- tokens/bearer-token_test.go | 48 ++++++++++++++++++++++++++------- 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 6a4d428..4c20546 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -82,11 +82,13 @@ func TestIntegration(t *testing.T) { CID, err := createContainer(ctx, t, clientPool, ownerID) require.NoError(t, err, version) - token := makeBearerToken(t, key, ownerID, version) + jsonToken, binaryToken := makeBearerTokens(t, key, ownerID, version) t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) }) - t.Run("put with bearer token in header"+version, func(t *testing.T) { putWithBearerTokenInHeader(ctx, t, clientPool, CID, token) }) - t.Run("put with bearer token in cookie"+version, func(t *testing.T) { putWithBearerTokenInCookie(ctx, t, clientPool, CID, token) }) + t.Run("put with json bearer token in header"+version, func(t *testing.T) { putWithBearerTokenInHeader(ctx, t, clientPool, CID, jsonToken) }) + t.Run("put with json bearer token in cookie"+version, func(t *testing.T) { putWithBearerTokenInCookie(ctx, t, clientPool, CID, jsonToken) }) + t.Run("put with binary bearer token in header"+version, func(t *testing.T) { putWithBearerTokenInHeader(ctx, t, clientPool, CID, binaryToken) }) + t.Run("put with binary bearer token in cookie"+version, func(t *testing.T) { putWithBearerTokenInCookie(ctx, t, clientPool, CID, binaryToken) }) t.Run("put with duplicate keys "+version, func(t *testing.T) { putWithDuplicateKeys(t, CID) }) t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, ownerID, CID) }) t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID) }) @@ -545,7 +547,7 @@ func registerUser(t *testing.T, ctx context.Context, aioContainer testcontainers require.NoError(t, err) } -func makeBearerToken(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string) string { +func makeBearerTokens(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string) (jsonTokenBase64, binaryTokenBase64 string) { tkn := new(bearer.Token) tkn.ForUser(ownerID) tkn.SetExp(10000) @@ -559,10 +561,16 @@ func makeBearerToken(t *testing.T, key *keys.PrivateKey, ownerID user.ID, versio err := tkn.Sign(key.PrivateKey) require.NoError(t, err) - t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) - require.NotEmpty(t, t64) + jsonToken, err := tkn.MarshalJSON() + require.NoError(t, err) - return t64 + jsonTokenBase64 = base64.StdEncoding.EncodeToString(jsonToken) + binaryTokenBase64 = base64.StdEncoding.EncodeToString(tkn.Marshal()) + + require.NotEmpty(t, jsonTokenBase64) + require.NotEmpty(t, binaryTokenBase64) + + return } func makeTempWallet(t *testing.T, key *keys.PrivateKey, path string) { diff --git a/tokens/bearer-token.go b/tokens/bearer-token.go index 880a100..24ffcbe 100644 --- a/tokens/bearer-token.go +++ b/tokens/bearer-token.go @@ -82,14 +82,22 @@ func fetchBearerToken(ctx *fasthttp.RequestCtx) (*bearer.Token, error) { tkn = new(bearer.Token) ) for _, parse := range []fromHandler{BearerTokenFromHeader, BearerTokenFromCookie} { - if buf = parse(&ctx.Request.Header); buf == nil { + buf = parse(&ctx.Request.Header) + if buf == nil { continue - } else if data, err := base64.StdEncoding.DecodeString(string(buf)); err != nil { + } + + data, err := base64.StdEncoding.DecodeString(string(buf)) + if err != nil { lastErr = fmt.Errorf("can't base64-decode bearer token: %w", err) continue - } else if err = tkn.Unmarshal(data); err != nil { - lastErr = fmt.Errorf("can't unmarshal bearer token: %w", err) - continue + } + + if err = tkn.Unmarshal(data); err != nil { + if err = tkn.UnmarshalJSON(data); err != nil { + lastErr = fmt.Errorf("can't unmarshal bearer token: %w", err) + continue + } } return tkn, nil diff --git a/tokens/bearer-token_test.go b/tokens/bearer-token_test.go index 6fb3bf4..60e9ea2 100644 --- a/tokens/bearer-token_test.go +++ b/tokens/bearer-token_test.go @@ -98,8 +98,14 @@ func TestFetchBearerToken(t *testing.T) { tkn := new(bearer.Token) tkn.ForUser(uid) - t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) - require.NotEmpty(t, t64) + jsonToken, err := tkn.MarshalJSON() + require.NoError(t, err) + + jsonTokenBase64 := base64.StdEncoding.EncodeToString(jsonToken) + binaryTokenBase64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) + + require.NotEmpty(t, jsonTokenBase64) + require.NotEmpty(t, binaryTokenBase64) cases := []struct { name string @@ -143,25 +149,47 @@ func TestFetchBearerToken(t *testing.T) { error: "can't unmarshal bearer token", }, { - name: "bad header, but good cookie", + name: "bad header, but good cookie with binary token", header: "dGVzdAo=", - cookie: t64, + cookie: binaryTokenBase64, expect: tkn, }, { - name: "bad cookie, but good header", - header: t64, + name: "bad cookie, but good header with binary token", + header: binaryTokenBase64, cookie: "dGVzdAo=", expect: tkn, }, { - name: "ok for header", - header: t64, + name: "bad header, but good cookie with json token", + header: "dGVzdAo=", + cookie: jsonTokenBase64, expect: tkn, }, { - name: "ok for cookie", - cookie: t64, + name: "bad cookie, but good header with json token", + header: jsonTokenBase64, + cookie: "dGVzdAo=", + expect: tkn, + }, + { + name: "ok for header with binary token", + header: binaryTokenBase64, + expect: tkn, + }, + { + name: "ok for cookie with binary token", + cookie: binaryTokenBase64, + expect: tkn, + }, + { + name: "ok for header with json token", + header: jsonTokenBase64, + expect: tkn, + }, + { + name: "ok for cookie with json token", + cookie: jsonTokenBase64, expect: tkn, }, } From e1b670a727529cb2232af4cedd1f6dc26d357559 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Thu, 9 Jan 2025 10:48:07 +0300 Subject: [PATCH 498/548] [#192] Build and host OCI images on our own infra Similar to https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/issues/587 this PR introduces a CI pipeline that builds Docker images and pushes them to our selfhosted registry. Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/oci-image.yml | 27 +++++++++++++++++++++++++++ Makefile | 2 +- README.md | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 .forgejo/workflows/oci-image.yml diff --git a/.forgejo/workflows/oci-image.yml b/.forgejo/workflows/oci-image.yml new file mode 100644 index 0000000..c5c0a2e --- /dev/null +++ b/.forgejo/workflows/oci-image.yml @@ -0,0 +1,27 @@ +on: + pull_request: + push: + workflow_dispatch: + +jobs: + image: + name: OCI image + runs-on: docker + container: git.frostfs.info/truecloudlab/env:oci-image-builder-bookworm + steps: + - name: Clone git repo + uses: actions/checkout@v3 + + - name: Build OCI image + run: make image + + - name: Push image to OCI registry + run: | + echo "$REGISTRY_PASSWORD" \ + | docker login --username truecloudlab --password-stdin git.frostfs.info + make image-push + if: >- + startsWith(github.ref, 'refs/tags/v') && + (github.event_name == 'workflow_dispatch' || github.event_name == 'push') + env: + REGISTRY_PASSWORD: ${{secrets.FORGEJO_OCI_REGISTRY_PUSH_TOKEN}} diff --git a/Makefile b/Makefile index c1f4f50..5b9e5bf 100755 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ LINT_VERSION ?= 1.60.3 TRUECLOUDLAB_LINT_VERSION ?= 0.0.6 BUILD ?= $(shell date -u --iso=seconds) -HUB_IMAGE ?= truecloudlab/frostfs-http-gw +HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs-http-gw HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" METRICS_DUMP_OUT ?= ./metrics-dump.json diff --git a/README.md b/README.md index adf793c..9c17c2a 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ version Show current version ``` Or you can also use a [Docker -image](https://hub.docker.com/r/truecloudlab/frostfs-http-gw) provided for the released +image](https://git.frostfs.info/TrueCloudLab/-/packages/container/frostfs-http-gw) provided for the released (and occasionally unreleased) versions of the gateway (`:latest` points to the latest stable release). From 1db62f9d953e79ca269951215280adb45ee12a53 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Fri, 20 Dec 2024 09:45:42 +0300 Subject: [PATCH 499/548] [#185] Update SDK to support new tree/pool version Signed-off-by: Marina Biryukova --- cmd/http-gw/app.go | 6 ++- cmd/http-gw/settings.go | 58 +++++++++++++++--------- config/config.env | 4 ++ config/config.yaml | 5 +++ docs/gate-configuration.md | 14 +++--- go.mod | 13 +++++- go.sum | 36 ++++++++++++++- internal/cache/buckets.go | 51 ++++++++++++++++++--- internal/cache/netmap.go | 65 +++++++++++++++++++++++++++ internal/data/info.go | 2 + internal/handler/handler.go | 1 + internal/handler/handler_test.go | 2 +- internal/logs/logs.go | 1 + internal/service/frostfs/frostfs.go | 10 +++++ internal/service/frostfs/source.go | 69 +++++++++++++++++++++++++++++ 15 files changed, 299 insertions(+), 38 deletions(-) create mode 100644 internal/cache/netmap.go create mode 100644 internal/service/frostfs/source.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 23a752a..3386536 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -65,6 +65,7 @@ type ( services []*metrics.Service settings *appSettings loggerSettings *loggerSettings + bucketCache *cache.BucketCache servers []Server unbindServers []ServerInfo @@ -134,6 +135,7 @@ func newApp(ctx context.Context, v *viper.Viper) App { loggerSettings: logSettings, webServer: new(fasthttp.Server), webDone: make(chan struct{}), + bucketCache: cache.NewBucketCache(getBucketCacheOptions(v, log.logger), v.GetBool(cfgFeaturesTreePoolNetmapSupport)), } a.initAppSettings() @@ -151,7 +153,7 @@ func newApp(ctx context.Context, v *viper.Viper) App { a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - a.pool, a.treePool, a.key = getPools(ctx, a.log, a.cfg, a.settings.dialerSource) + a.initPools(ctx) var owner user.ID user.IDFromKey(&owner, a.key.PrivateKey.PublicKey) @@ -839,7 +841,7 @@ func (a *app) AppParams() *handler.AppParams { FrostFS: frostfs.NewFrostFS(a.pool), Owner: a.owner, Resolver: a.resolver, - Cache: cache.NewBucketCache(getCacheOptions(a.cfg, a.log)), + Cache: a.bucketCache, } } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 62ef83e..691e9ba 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -17,12 +17,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "git.frostfs.info/TrueCloudLab/zapjournald" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/ssgreg/journald" @@ -144,6 +144,7 @@ const ( // Caching. cfgBucketsCacheLifetime = "cache.buckets.lifetime" cfgBucketsCacheSize = "cache.buckets.size" + cfgNetmapCacheLifetime = "cache.netmap.lifetime" // Bucket resolving options. cfgResolveNamespaceHeader = "resolve_bucket.namespace_header" @@ -166,6 +167,7 @@ const ( // Feature. cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" + cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support" // Command line args. cmdHelp = "help" @@ -611,10 +613,10 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { return servers } -func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper, dialSource *internalnet.DialerSource) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) { - key, err := getFrostFSKey(cfg, logger) +func (a *app) initPools(ctx context.Context) { + key, err := getFrostFSKey(a.cfg, a.log) if err != nil { - logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) + a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) } var prm pool.InitParameters @@ -622,77 +624,83 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper, dialSou prm.SetKey(&key.PrivateKey) prmTree.SetKey(key) - logger.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) + a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) - for _, peer := range fetchPeers(logger, cfg) { + for _, peer := range fetchPeers(a.log, a.cfg) { prm.AddNode(peer) prmTree.AddNode(peer) } - connTimeout := cfg.GetDuration(cfgConTimeout) + connTimeout := a.cfg.GetDuration(cfgConTimeout) if connTimeout <= 0 { connTimeout = defaultConnectTimeout } prm.SetNodeDialTimeout(connTimeout) prmTree.SetNodeDialTimeout(connTimeout) - streamTimeout := cfg.GetDuration(cfgStreamTimeout) + streamTimeout := a.cfg.GetDuration(cfgStreamTimeout) if streamTimeout <= 0 { streamTimeout = defaultStreamTimeout } prm.SetNodeStreamTimeout(streamTimeout) prmTree.SetNodeStreamTimeout(streamTimeout) - healthCheckTimeout := cfg.GetDuration(cfgReqTimeout) + healthCheckTimeout := a.cfg.GetDuration(cfgReqTimeout) if healthCheckTimeout <= 0 { healthCheckTimeout = defaultRequestTimeout } prm.SetHealthcheckTimeout(healthCheckTimeout) prmTree.SetHealthcheckTimeout(healthCheckTimeout) - rebalanceInterval := cfg.GetDuration(cfgRebalance) + rebalanceInterval := a.cfg.GetDuration(cfgRebalance) if rebalanceInterval <= 0 { rebalanceInterval = defaultRebalanceTimer } prm.SetClientRebalanceInterval(rebalanceInterval) prmTree.SetClientRebalanceInterval(rebalanceInterval) - errorThreshold := cfg.GetUint32(cfgPoolErrorThreshold) + errorThreshold := a.cfg.GetUint32(cfgPoolErrorThreshold) if errorThreshold <= 0 { errorThreshold = defaultPoolErrorThreshold } prm.SetErrorThreshold(errorThreshold) - prm.SetLogger(logger) - prmTree.SetLogger(logger) + prm.SetLogger(a.log) + prmTree.SetLogger(a.log) - prmTree.SetMaxRequestAttempts(cfg.GetInt(cfgTreePoolMaxAttempts)) + prmTree.SetMaxRequestAttempts(a.cfg.GetInt(cfgTreePoolMaxAttempts)) interceptors := []grpc.DialOption{ grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), - grpc.WithContextDialer(dialSource.GrpcContextDialer()), + grpc.WithContextDialer(a.settings.dialerSource.GrpcContextDialer()), } prm.SetGRPCDialOptions(interceptors...) prmTree.SetGRPCDialOptions(interceptors...) p, err := pool.NewPool(prm) if err != nil { - logger.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) } if err = p.Dial(ctx); err != nil { - logger.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + } + + if a.cfg.GetBool(cfgFeaturesTreePoolNetmapSupport) { + prmTree.SetNetMapInfoSource(frostfs.NewSource(frostfs.NewFrostFS(p), cache.NewNetmapCache(getNetmapCacheOptions(a.cfg, a.log)), a.bucketCache, a.log)) } treePool, err := treepool.NewPool(prmTree) if err != nil { - logger.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) } if err = treePool.Dial(ctx); err != nil { - logger.Fatal(logs.FailedToDialTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err)) } - return p, treePool, key + a.pool = p + a.treePool = treePool + a.key = key } func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam { @@ -733,7 +741,7 @@ func fetchSoftMemoryLimit(cfg *viper.Viper) int64 { return int64(softMemoryLimit) } -func getCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { +func getBucketCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { cacheCfg := cache.DefaultBucketConfig(l) cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgBucketsCacheLifetime, cacheCfg.Lifetime) @@ -742,6 +750,14 @@ func getCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { return cacheCfg } +func getNetmapCacheOptions(v *viper.Viper, l *zap.Logger) *cache.NetmapCacheConfig { + cacheCfg := cache.DefaultNetmapConfig(l) + + cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgNetmapCacheLifetime, cacheCfg.Lifetime) + + return cacheCfg +} + func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration { if v.IsSet(cfgEntry) { lifetime := v.GetDuration(cfgEntry) diff --git a/config/config.env b/config/config.env index 2822357..171889f 100644 --- a/config/config.env +++ b/config/config.env @@ -121,6 +121,8 @@ HTTP_GW_FROSTFS_BUFFER_MAX_SIZE_FOR_PUT=1048576 # Cache which contains mapping of bucket name to bucket info HTTP_GW_CACHE_BUCKETS_LIFETIME=1m HTTP_GW_CACHE_BUCKETS_SIZE=1000 +# Cache which stores netmap +HTTP_GW_CACHE_NETMAP_LIFETIME=1m # Header to determine zone to resolve bucket name HTTP_GW_RESOLVE_BUCKET_NAMESPACE_HEADER=X-Frostfs-Namespace @@ -162,3 +164,5 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl # Enable using fallback path to search for a object by attribute HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false +# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service +HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true diff --git a/config/config.yaml b/config/config.yaml index 6296bd9..eee84e5 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -143,6 +143,9 @@ cache: buckets: lifetime: 1m size: 1000 + # Cache which stores netmap + netmap: + lifetime: 1m resolve_bucket: namespace_header: X-Frostfs-Namespace @@ -176,3 +179,5 @@ multinet: features: # Enable using fallback path to search for a object by attribute enable_filepath_fallback: false + # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service + tree_pool_netmap_support: true diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 7476f5d..ce7c0c7 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -339,12 +339,14 @@ cache: buckets: lifetime: 1m size: 1000 - + netmap: + lifetime: 1m ``` -| Parameter | Type | Default value | Description | -|-----------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------------------------| -| `buckets` | [Cache config](#cache-subsection) | `lifetime: 60s`
`size: 1000` | Cache which contains mapping of bucket name to bucket info. | +| Parameter | Type | Default value | Description | +|-----------|-----------------------------------|---------------------------------|---------------------------------------------------------------------------| +| `buckets` | [Cache config](#cache-subsection) | `lifetime: 60s`
`size: 1000` | Cache which contains mapping of bucket name to bucket info. | +| `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. | #### `cache` subsection @@ -465,8 +467,10 @@ Contains parameters for enabling features. ```yaml features: enable_filepath_fallback: true + tree_pool_netmap_support: true ``` | Parameter | Type | SIGHUP reload | Default value | Description | -| ----------------------------------- | ------ | ------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +|-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. | +| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | diff --git a/go.mod b/go.mod index 3dd27b8..0b74841 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 @@ -65,16 +65,25 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/ipfs/go-cid v0.0.7 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.16.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/sys/mount v0.3.2 // indirect github.com/moby/sys/mountinfo v0.6.1 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multiaddr v0.14.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect @@ -89,6 +98,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect github.com/sirupsen/logrus v1.8.1 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -116,4 +126,5 @@ require ( google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 7f43c86..06bdd7d 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d h1:FpXI+mOrmJk3t2MKQFZuhLjCHDyDeo5rtP1WXl7gUWc= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae h1:7gvuOTmS3oaOM79JkHWWlsvGqIRqsum5KnOI1TYqfn0= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae/go.mod h1:dbWUc5jOBTXVvssCLCYxkkSTL9jgLr1KruGP2FMAfiM= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= @@ -519,6 +519,8 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -544,6 +546,8 @@ github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -582,6 +586,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -610,9 +618,28 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU= +github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -762,6 +789,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= @@ -1121,6 +1150,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1407,6 +1437,8 @@ k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/cache/buckets.go b/internal/cache/buckets.go index f8e6d88..2fa8f25 100644 --- a/internal/cache/buckets.go +++ b/internal/cache/buckets.go @@ -6,14 +6,16 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/bluele/gcache" "go.uber.org/zap" ) // BucketCache contains cache with objects and the lifetime of cache entries. type BucketCache struct { - cache gcache.Cache - logger *zap.Logger + cache gcache.Cache + cidCache gcache.Cache + logger *zap.Logger } // Config stores expiration params for cache. @@ -40,14 +42,45 @@ func DefaultBucketConfig(logger *zap.Logger) *Config { } // NewBucketCache creates an object of BucketCache. -func NewBucketCache(config *Config) *BucketCache { - gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() - return &BucketCache{cache: gc, logger: config.Logger} +func NewBucketCache(config *Config, cidCache bool) *BucketCache { + cache := &BucketCache{ + cache: gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build(), + logger: config.Logger, + } + + if cidCache { + cache.cidCache = gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + } + return cache } // Get returns a cached object. func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo { - entry, err := o.cache.Get(formKey(ns, bktName)) + return o.get(formKey(ns, bktName)) +} + +func (o *BucketCache) GetByCID(cnrID cid.ID) *data.BucketInfo { + if o.cidCache == nil { + return nil + } + + entry, err := o.cidCache.Get(cnrID) + if err != nil { + return nil + } + + key, ok := entry.(string) + if !ok { + o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", key))) + return nil + } + + return o.get(key) +} + +func (o *BucketCache) get(key string) *data.BucketInfo { + entry, err := o.cache.Get(key) if err != nil { return nil } @@ -64,6 +97,12 @@ func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo { // Put puts an object to cache. func (o *BucketCache) Put(bkt *data.BucketInfo) error { + if o.cidCache != nil { + if err := o.cidCache.Set(bkt.CID, formKey(bkt.Zone, bkt.Name)); err != nil { + return err + } + } + return o.cache.Set(formKey(bkt.Zone, bkt.Name), bkt) } diff --git a/internal/cache/netmap.go b/internal/cache/netmap.go new file mode 100644 index 0000000..6d91fe7 --- /dev/null +++ b/internal/cache/netmap.go @@ -0,0 +1,65 @@ +package cache + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/bluele/gcache" + "go.uber.org/zap" +) + +type ( + // NetmapCache provides cache for netmap. + NetmapCache struct { + cache gcache.Cache + logger *zap.Logger + } + + // NetmapCacheConfig stores expiration params for cache. + NetmapCacheConfig struct { + Lifetime time.Duration + Logger *zap.Logger + } +) + +const ( + DefaultNetmapCacheLifetime = time.Minute + netmapCacheSize = 1 + netmapKey = "netmap" +) + +// DefaultNetmapConfig returns new default cache expiration values. +func DefaultNetmapConfig(logger *zap.Logger) *NetmapCacheConfig { + return &NetmapCacheConfig{ + Lifetime: DefaultNetmapCacheLifetime, + Logger: logger, + } +} + +// NewNetmapCache creates an object of NetmapCache. +func NewNetmapCache(config *NetmapCacheConfig) *NetmapCache { + gc := gcache.New(netmapCacheSize).LRU().Expiration(config.Lifetime).Build() + return &NetmapCache{cache: gc, logger: config.Logger} +} + +func (c *NetmapCache) Get() *netmap.NetMap { + entry, err := c.cache.Get(netmapKey) + if err != nil { + return nil + } + + result, ok := entry.(netmap.NetMap) + if !ok { + c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result))) + return nil + } + + return &result +} + +func (c *NetmapCache) Put(nm netmap.NetMap) error { + return c.cache.Set(netmapKey, nm) +} diff --git a/internal/data/info.go b/internal/data/info.go index d99ca49..f5c80d6 100644 --- a/internal/data/info.go +++ b/internal/data/info.go @@ -2,6 +2,7 @@ package data import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) type BucketInfo struct { @@ -9,4 +10,5 @@ type BucketInfo struct { Zone string // container zone from system attribute CID cid.ID HomomorphicHashDisabled bool + PlacementPolicy netmap.PlacementPolicy } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 3805c2d..1150f45 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -365,6 +365,7 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket } bktInfo.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(*res) + bktInfo.PlacementPolicy = res.PlacementPolicy() return bktInfo, err } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 14f9c98..e1bc010 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -142,7 +142,7 @@ func prepareHandlerContext() (*handlerContext, error) { Size: 1, Lifetime: 1, Logger: logger, - }), + }, false), } treeMock := newTreeService() diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 7a04064..f9b13b1 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -93,4 +93,5 @@ const ( FailedToLoadMultinetConfig = "failed to load multinet config" MultinetConfigWontBeUpdated = "multinet config won't be updated" ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" + CouldntCacheNetmap = "couldn't cache netmap" ) diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index c7e56a4..b218976 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" @@ -173,6 +174,15 @@ func (x *FrostFS) GetEpochDurations(ctx context.Context) (*utils.EpochDurations, return res, nil } +func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) { + netmapSnapshot, err := x.pool.NetMapSnapshot(ctx) + if err != nil { + return netmapSnapshot, handleObjectError("get netmap via connection pool", err) + } + + return netmapSnapshot, nil +} + // ResolverFrostFS represents virtual connection to the FrostFS network. // It implements resolver.FrostFS. type ResolverFrostFS struct { diff --git a/internal/service/frostfs/source.go b/internal/service/frostfs/source.go new file mode 100644 index 0000000..de6c681 --- /dev/null +++ b/internal/service/frostfs/source.go @@ -0,0 +1,69 @@ +package frostfs + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "go.uber.org/zap" +) + +type Source struct { + frostFS *FrostFS + netmapCache *cache.NetmapCache + bucketCache *cache.BucketCache + log *zap.Logger +} + +func NewSource(frostFS *FrostFS, netmapCache *cache.NetmapCache, bucketCache *cache.BucketCache, log *zap.Logger) *Source { + return &Source{ + frostFS: frostFS, + netmapCache: netmapCache, + bucketCache: bucketCache, + log: log, + } +} + +func (s *Source) NetMapSnapshot(ctx context.Context) (netmap.NetMap, error) { + cachedNetmap := s.netmapCache.Get() + if cachedNetmap != nil { + return *cachedNetmap, nil + } + + netmapSnapshot, err := s.frostFS.NetmapSnapshot(ctx) + if err != nil { + return netmap.NetMap{}, fmt.Errorf("get netmap: %w", err) + } + + if err = s.netmapCache.Put(netmapSnapshot); err != nil { + s.log.Warn(logs.CouldntCacheNetmap, zap.Error(err)) + } + + return netmapSnapshot, nil +} + +func (s *Source) PlacementPolicy(ctx context.Context, cnrID cid.ID) (netmap.PlacementPolicy, error) { + info := s.bucketCache.GetByCID(cnrID) + if info != nil { + return info.PlacementPolicy, nil + } + + prm := handler.PrmContainer{ + ContainerID: cnrID, + } + res, err := s.frostFS.Container(ctx, prm) + if err != nil { + return netmap.PlacementPolicy{}, fmt.Errorf("get container: %w", err) + } + + // We don't put container back to the cache to keep cache + // coherent to the requests made by users. FrostFS Source + // is being used by SDK Tree Pool and it should not fill cache + // with possibly irrelevant container values. + + return res.PlacementPolicy(), nil +} From f0c999d9a28e9b6cf3419c1147178ce8626f4c6e Mon Sep 17 00:00:00 2001 From: Aleksey Kravchenko Date: Tue, 24 Dec 2024 18:42:02 +0300 Subject: [PATCH 500/548] [#188] Improve content-type detector Signed-off-by: Aleksey Kravchenko --- internal/handler/head.go | 15 ++++++- internal/handler/reader.go | 21 ++++++++-- internal/handler/reader_test.go | 71 ++++++++++++++++++++++++++------- 3 files changed, 87 insertions(+), 20 deletions(-) diff --git a/internal/handler/head.go b/internal/handler/head.go index f2e9f38..da96eff 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -45,7 +45,11 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid } req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10)) - var contentType string + var ( + contentType string + filename string + filepath string + ) for _, attr := range obj.Attributes() { key := attr.Key() val := attr.Value() @@ -69,8 +73,15 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid req.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat)) case object.AttributeContentType: contentType = val + case object.AttributeFilePath: + filepath = val + case object.AttributeFileName: + filename = val } } + if filename == "" { + filename = filepath + } idsToResponse(&req.Response, obj) @@ -85,7 +96,7 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid } return h.frostfs.RangeObject(ctx, prmRange) - }) + }, filename) if err != nil && err != io.EOF { req.handleFrostFSErr(err, start) return diff --git a/internal/handler/reader.go b/internal/handler/reader.go index 50121c9..60067ab 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -4,9 +4,11 @@ import ( "bytes" "context" "io" + "mime" "net/http" "path" "strconv" + "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" @@ -25,7 +27,7 @@ type readCloser struct { // initializes io.Reader with the limited size and detects Content-Type from it. // Returns r's error directly. Also returns the processed data. -func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (string, []byte, error) { +func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error), filename string) (string, []byte, error) { if maxSize > sizeToDetectType { maxSize = sizeToDetectType } @@ -44,7 +46,20 @@ func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (str buf = buf[:n] - return http.DetectContentType(buf), buf, err // to not lose io.EOF + contentType := http.DetectContentType(buf) + + // Since the detector detects the "text/plain" content type for various types of text files, + // including CSS, JavaScript, and CSV files, + // we'll determine the final content type based on the file's extension. + if strings.HasPrefix(contentType, "text/plain") { + ext := path.Ext(filename) + // If the file doesn't have a file extension, we'll keep the content type as is. + if len(ext) > 0 { + contentType = mime.TypeByExtension(ext) + } + } + + return contentType, buf, err // to not lose io.EOF } type getMultiobjectBodyParams struct { @@ -128,7 +143,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A contentType, payloadHead, err = readContentType(payloadSize, func(uint64) (io.Reader, error) { return payload, nil - }) + }, filename) if err != nil && err != io.EOF { req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) response.Error(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) diff --git a/internal/handler/reader_test.go b/internal/handler/reader_test.go index c63a734..f867677 100644 --- a/internal/handler/reader_test.go +++ b/internal/handler/reader_test.go @@ -10,39 +10,80 @@ import ( "github.com/stretchr/testify/require" ) +const ( + txtContentType = "text/plain; charset=utf-8" + cssContentType = "text/css; charset=utf-8" + htmlContentType = "text/html; charset=utf-8" + javascriptContentType = "text/javascript; charset=utf-8" + + htmlBody = "Test Html" +) + func TestDetector(t *testing.T) { - txtContentType := "text/plain; charset=utf-8" sb := strings.Builder{} for i := 0; i < 10; i++ { sb.WriteString("Some txt content. Content-Type must be detected properly by detector.") } for _, tc := range []struct { - Name string - ContentType string - Expected string + Name string + ExpectedContentType string + Content string + FileName string }{ { - Name: "less than 512b", - ContentType: txtContentType, - Expected: sb.String()[:256], + Name: "less than 512b", + ExpectedContentType: txtContentType, + Content: sb.String()[:256], + FileName: "test.txt", }, { - Name: "more than 512b", - ContentType: txtContentType, - Expected: sb.String(), + Name: "more than 512b", + ExpectedContentType: txtContentType, + Content: sb.String(), + FileName: "test.txt", + }, + { + Name: "css content type", + ExpectedContentType: cssContentType, + Content: sb.String(), + FileName: "test.css", + }, + { + Name: "javascript content type", + ExpectedContentType: javascriptContentType, + Content: sb.String(), + FileName: "test.js", + }, + { + Name: "html content type by file content", + ExpectedContentType: htmlContentType, + Content: htmlBody, + FileName: "test.detect-by-content", + }, + { + Name: "html content type by file extension", + ExpectedContentType: htmlContentType, + Content: sb.String(), + FileName: "test.html", + }, + { + Name: "empty file extension", + ExpectedContentType: txtContentType, + Content: sb.String(), + FileName: "test", }, } { t.Run(tc.Name, func(t *testing.T) { - contentType, data, err := readContentType(uint64(len(tc.Expected)), + contentType, data, err := readContentType(uint64(len(tc.Content)), func(uint64) (io.Reader, error) { - return strings.NewReader(tc.Expected), nil - }, + return strings.NewReader(tc.Content), nil + }, tc.FileName, ) require.NoError(t, err) - require.Equal(t, tc.ContentType, contentType) - require.True(t, strings.HasPrefix(tc.Expected, string(data))) + require.Equal(t, tc.ExpectedContentType, contentType) + require.True(t, strings.HasPrefix(tc.Content, string(data))) }) } } From 4b782cf1247fcf2cac42d1c58a610c4c98d8698e Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 9 Jan 2025 12:28:38 +0300 Subject: [PATCH 501/548] [#187] Add handling quota limit reached error The Access Denied status may be received from APE due to exceeding the quota. In this situation, you need to return the appropriate status code. Signed-off-by: Roman Loginov --- CHANGELOG.md | 3 + cmd/http-gw/app.go | 21 +++--- internal/handler/download.go | 5 +- internal/handler/handler.go | 13 ++-- internal/handler/reader.go | 3 +- internal/handler/upload.go | 13 ++-- internal/handler/utils.go | 45 +++++++++++-- internal/service/frostfs/frostfs.go | 4 ++ internal/service/frostfs/frostfs_test.go | 83 ++++++++++++++++++++++++ response/utils.go | 41 ------------ 10 files changed, 156 insertions(+), 75 deletions(-) create mode 100644 internal/service/frostfs/frostfs_test.go delete mode 100644 response/utils.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e528b8b..fd37815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ This document outlines major changes between releases. ## [Unreleased] +### Added +- Add handling quota limit reached error (#187) + ## [0.32.0] - Khumbu - 2024-12-20 ### Fixed diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 3386536..e34386c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -25,7 +25,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" @@ -636,28 +635,28 @@ func (a *app) stopServices() { } } -func (a *app) configureRouter(handler *handler.Handler) { +func (a *app) configureRouter(h *handler.Handler) { r := router.New() r.RedirectTrailingSlash = true r.NotFound = func(r *fasthttp.RequestCtx) { - response.Error(r, "Not found", fasthttp.StatusNotFound) + handler.ResponseError(r, "Not found", fasthttp.StatusNotFound) } r.MethodNotAllowed = func(r *fasthttp.RequestCtx) { - response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) + handler.ResponseError(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.addMiddlewares(handler.Upload)) + r.POST("/upload/{cid}", a.addMiddlewares(h.Upload)) r.OPTIONS("/upload/{cid}", a.addPreflight()) a.log.Info(logs.AddedPathUploadCid) - r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(handler.DownloadByAddressOrBucketName)) - r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(handler.HeadByAddressOrBucketName)) + r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(h.DownloadByAddressOrBucketName)) + r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(h.HeadByAddressOrBucketName)) r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetCidOid) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(handler.DownloadByAttribute)) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(handler.HeadByAttribute)) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.DownloadByAttribute)) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.HeadByAttribute)) r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(handler.DownloadZipped)) + r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZipped)) r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) a.log.Info(logs.AddedPathZipCidPrefix) @@ -800,7 +799,7 @@ func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { log := utils.GetReqLogOrDefault(reqCtx, a.log) log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err)) - response.Error(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) + handler.ResponseError(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } utils.SetContextToRequest(appCtx, req) diff --git a/internal/handler/download.go b/internal/handler/download.go index de27fa3..8766f0c 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -120,7 +119,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { prefix, err := url.QueryUnescape(prefix) if err != nil { log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) - response.Error(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -135,7 +134,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) - response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 1150f45..2f1c6ad 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -140,6 +139,8 @@ var ( ErrAccessDenied = errors.New("access denied") // ErrGatewayTimeout is returned from FrostFS in case of timeout, deadline exceeded etc. ErrGatewayTimeout = errors.New("gateway timeout") + // ErrQuotaLimitReached is returned from FrostFS in case of quota exceeded. + ErrQuotaLimitReached = errors.New("quota limit reached") ) // FrostFS represents virtual connection to FrostFS network. @@ -210,7 +211,7 @@ func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path } if foundOID.IsDeleteMarker { log.Error(logs.ObjectWasDeleted) - response.Error(c, "object deleted", fasthttp.StatusNotFound) + ResponseError(c, "object deleted", fasthttp.StatusNotFound) return } @@ -230,14 +231,14 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte key, err := url.QueryUnescape(key) if err != nil { log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), zap.Error(err)) - response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) return } val, err = url.QueryUnescape(val) if err != nil { log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), zap.Error(err)) - response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -252,11 +253,11 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte objID, err := h.findObjectByAttribute(ctx, log, bktInfo.CID, key, val) if err != nil { if errors.Is(err, io.EOF) { - response.Error(c, err.Error(), fasthttp.StatusNotFound) + ResponseError(c, err.Error(), fasthttp.StatusNotFound) return } - response.Error(c, err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/reader.go b/internal/handler/reader.go index 60067ab..cbd8294 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -12,7 +12,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -146,7 +145,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A }, filename) if err != nil && err != io.EOF { req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) - response.Error(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 867025d..9493635 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -9,7 +9,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" @@ -81,14 +80,14 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) - response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } filtered, err := filterHeaders(log, &c.Request.Header) if err != nil { log.Error(logs.CouldNotProcessHeaders, zap.Error(err)) - response.Error(c, err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, err.Error(), fasthttp.StatusBadRequest) return } @@ -103,7 +102,7 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { if err = utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) - response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -157,7 +156,7 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) - response.Error(c, "could not encode response", fasthttp.StatusBadRequest) + ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) return } @@ -179,11 +178,11 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { } func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *zap.Logger) { - statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err) + statusCode, msg, additionalFields := formErrorResponse("could not store file in frostfs", err) logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) - response.Error(r, msg, statusCode) + ResponseError(r, msg, statusCode) } func (h *Handler) fetchBearerToken(ctx context.Context) *bearer.Token { diff --git a/internal/handler/utils.go b/internal/handler/utils.go index d09ed23..7fdd396 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -2,14 +2,16 @@ package handler import ( "context" + "errors" + "fmt" "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -27,11 +29,11 @@ func (r *request) handleFrostFSErr(err error, start time.Time) { zap.Stringer("elapsed", time.Since(start)), zap.Error(err), } - statusCode, msg, additionalFields := response.FormErrorResponse("could not receive object", err) + statusCode, msg, additionalFields := formErrorResponse("could not receive object", err) logFields = append(logFields, additionalFields...) r.log.Error(logs.CouldNotReceiveObject, logFields...) - response.Error(r.RequestCtx, msg, statusCode) + ResponseError(r.RequestCtx, msg, statusCode) } func bearerToken(ctx context.Context) *bearer.Token { @@ -79,10 +81,10 @@ func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { log.Error(logs.CouldntGetBucket, zap.Error(err)) if client.IsErrContainerNotFound(err) { - response.Error(c, "Not Found", fasthttp.StatusNotFound) + ResponseError(c, "Not Found", fasthttp.StatusNotFound) return } - response.Error(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) } func newAddress(cnr cid.ID, obj oid.ID) oid.Address { @@ -91,3 +93,36 @@ func newAddress(cnr cid.ID, obj oid.ID) oid.Address { addr.SetObject(obj) return addr } + +func ResponseError(r *fasthttp.RequestCtx, msg string, code int) { + r.Error(msg+"\n", code) +} + +func formErrorResponse(message string, err error) (int, string, []zap.Field) { + var ( + msg string + statusCode int + logFields []zap.Field + ) + + st := new(sdkstatus.ObjectAccessDenied) + + switch { + case errors.As(err, &st): + statusCode = fasthttp.StatusForbidden + reason := st.Reason() + msg = fmt.Sprintf("%s: %v: %s", message, err, reason) + logFields = append(logFields, zap.String("error_detail", reason)) + case errors.Is(err, ErrQuotaLimitReached): + statusCode = fasthttp.StatusConflict + msg = fmt.Sprintf("%s: %v", message, err) + case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err): + statusCode = fasthttp.StatusNotFound + msg = "Not Found" + default: + statusCode = fasthttp.StatusBadRequest + msg = fmt.Sprintf("%s: %v", message, err) + } + + return statusCode, msg, logFields +} diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index b218976..c6af526 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -215,6 +215,10 @@ func handleObjectError(msg string, err error) error { } if reason, ok := IsErrObjectAccessDenied(err); ok { + if strings.Contains(reason, "limit reached") { + return fmt.Errorf("%s: %w: %s", msg, handler.ErrQuotaLimitReached, reason) + } + return fmt.Errorf("%s: %w: %s", msg, handler.ErrAccessDenied, reason) } diff --git a/internal/service/frostfs/frostfs_test.go b/internal/service/frostfs/frostfs_test.go new file mode 100644 index 0000000..e9b3329 --- /dev/null +++ b/internal/service/frostfs/frostfs_test.go @@ -0,0 +1,83 @@ +package frostfs + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestHandleObjectError(t *testing.T) { + msg := "some msg" + + t.Run("nil error", func(t *testing.T) { + err := handleObjectError(msg, nil) + require.Nil(t, err) + }) + + t.Run("simple access denied", func(t *testing.T) { + reason := "some reason" + inputErr := new(apistatus.ObjectAccessDenied) + inputErr.WriteReason(reason) + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrAccessDenied) + require.Contains(t, err.Error(), reason) + require.Contains(t, err.Error(), msg) + }) + + t.Run("access denied - quota reached", func(t *testing.T) { + reason := "Quota limit reached" + inputErr := new(apistatus.ObjectAccessDenied) + inputErr.WriteReason(reason) + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrQuotaLimitReached) + require.Contains(t, err.Error(), reason) + require.Contains(t, err.Error(), msg) + }) + + t.Run("simple timeout", func(t *testing.T) { + inputErr := errors.New("timeout") + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrGatewayTimeout) + require.Contains(t, err.Error(), inputErr.Error()) + require.Contains(t, err.Error(), msg) + }) + + t.Run("deadline exceeded", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + <-ctx.Done() + + err := handleObjectError(msg, ctx.Err()) + require.ErrorIs(t, err, handler.ErrGatewayTimeout) + require.Contains(t, err.Error(), ctx.Err().Error()) + require.Contains(t, err.Error(), msg) + }) + + t.Run("grpc deadline exceeded", func(t *testing.T) { + inputErr := fmt.Errorf("wrap grpc error: %w", status.Error(codes.DeadlineExceeded, "error")) + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrGatewayTimeout) + require.Contains(t, err.Error(), inputErr.Error()) + require.Contains(t, err.Error(), msg) + }) + + t.Run("unknown error", func(t *testing.T) { + inputErr := errors.New("unknown error") + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, inputErr) + require.Contains(t, err.Error(), msg) + }) +} diff --git a/response/utils.go b/response/utils.go deleted file mode 100644 index f233943..0000000 --- a/response/utils.go +++ /dev/null @@ -1,41 +0,0 @@ -package response - -import ( - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "github.com/valyala/fasthttp" - "go.uber.org/zap" -) - -func Error(r *fasthttp.RequestCtx, msg string, code int) { - r.Error(msg+"\n", code) -} - -func FormErrorResponse(message string, err error) (int, string, []zap.Field) { - var ( - msg string - statusCode int - logFields []zap.Field - ) - - st := new(sdkstatus.ObjectAccessDenied) - - switch { - case errors.As(err, &st): - statusCode = fasthttp.StatusForbidden - reason := st.Reason() - msg = fmt.Sprintf("%s: %v: %s", message, err, reason) - logFields = append(logFields, zap.String("error_detail", reason)) - case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err): - statusCode = fasthttp.StatusNotFound - msg = "Not Found" - default: - statusCode = fasthttp.StatusBadRequest - msg = fmt.Sprintf("%s: %v", message, err) - } - - return statusCode, msg, logFields -} From 1e82f64dfde297712006b1fea80fd186c1d0d455 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 21 Jan 2025 11:07:00 +0300 Subject: [PATCH 502/548] [#193] Enable integration tests in Forgejo Actions Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/tests.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 81d93dc..d4182ed 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -43,3 +43,19 @@ jobs: - name: Run tests run: make test + + integration: + name: Integration tests + runs-on: oci-runner + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.23' + + - name: Run integration tests + run: |- + podman-service.sh + make integration-test From 856e0ecf40fd659b5f9ebd182888cfa7a269bf26 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 21 Jan 2025 11:33:58 +0300 Subject: [PATCH 503/548] [#193] Update testcontainers to v0.35.0 Signed-off-by: Vitaliy Potyarkin --- cmd/http-gw/integration_test.go | 15 +- go.mod | 60 ++- go.sum | 819 ++++---------------------------- 3 files changed, 135 insertions(+), 759 deletions(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 4c20546..0c2bdf4 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -29,6 +29,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + docker "github.com/docker/docker/api/types/container" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/viper" @@ -434,11 +435,13 @@ func checkNamespaces(ctx context.Context, t *testing.T, clientPool *pool.Pool, o func createDockerContainer(ctx context.Context, t *testing.T, image string) testcontainers.Container { req := testcontainers.ContainerRequest{ - Image: image, - WaitingFor: wait.NewLogStrategy("aio container started").WithStartupTimeout(2 * time.Minute), - Name: "aio", - Hostname: "aio", - NetworkMode: "host", + Image: image, + WaitingFor: wait.NewLogStrategy("aio container started").WithStartupTimeout(2 * time.Minute), + Name: "aio", + Hostname: "aio", + HostConfigModifier: func(hc *docker.HostConfig) { + hc.NetworkMode = "host" + }, } aioC, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, @@ -539,7 +542,7 @@ func registerUser(t *testing.T, ctx context.Context, aioContainer testcontainers err := aioContainer.CopyFileToContainer(ctx, pathToWallet, "/usr/wallet.json", 644) require.NoError(t, err) - _, err = aioContainer.Exec(ctx, []string{ + _, _, err = aioContainer.Exec(ctx, []string{ "/usr/bin/frostfs-s3-authmate", "register-user", "--wallet", "/usr/wallet.json", "--rpc-endpoint", "http://localhost:30333", diff --git a/go.mod b/go.mod index 0b74841..46fe3bc 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,8 @@ require ( git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 - github.com/docker/go-units v0.4.0 + github.com/docker/docker v27.1.1+incompatible + github.com/docker/go-units v0.5.0 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.106.2 github.com/panjf2000/ants/v2 v2.5.0 @@ -18,7 +19,7 @@ require ( github.com/spf13/viper v1.15.0 github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.9.0 - github.com/testcontainers/testcontainers-go v0.13.0 + github.com/testcontainers/testcontainers-go v0.35.0 github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 github.com/valyala/fasthttp v1.34.0 go.opentelemetry.io/otel v1.28.0 @@ -26,56 +27,60 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/net v0.26.0 - golang.org/x/sys v0.22.0 + golang.org/x/sys v0.28.0 google.golang.org/grpc v1.66.2 ) require ( + dario.cat/mergo v1.0.0 // indirect git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e // indirect git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/Microsoft/hcsshim v0.9.2 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/VictoriaMetrics/easyproto v0.1.4 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/containerd/cgroups v1.0.3 // indirect - github.com/containerd/containerd v1.6.2 // indirect + github.com/containerd/containerd v1.7.18 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/docker v20.10.14+incompatible // indirect - github.com/docker/go-connections v0.4.0 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/ipfs/go-cid v0.0.7 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.16.4 // indirect + github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/sys/mount v0.3.2 // indirect - github.com/moby/sys/mountinfo v0.6.1 // indirect - github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect @@ -88,27 +93,32 @@ require ( github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.2 // indirect - github.com/opencontainers/runc v1.1.1 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect - github.com/sirupsen/logrus v1.8.1 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/twmb/murmur3 v1.1.8 // indirect - github.com/urfave/cli v1.22.5 // indirect + github.com/urfave/cli v1.22.12 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.9 // indirect - go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect @@ -116,10 +126,10 @@ require ( go.opentelemetry.io/otel/sdk v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/term v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect diff --git a/go.sum b/go.sum index 06bdd7d..e5bfc09 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -36,6 +35,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= @@ -55,357 +56,101 @@ git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjq git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 h1:HeY8n27VyPRQe49l/fzyVMkWEB2fsLJYKp64pwA7tz4= git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02/go.mod h1:rQFJJdEOV7KbbMtQYR2lNfiZk+ONRDJSbMCTWxKt8Fw= -github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= -github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= -github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= -github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= -github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= -github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= -github.com/Microsoft/hcsshim v0.9.2 h1:wB06W5aYFfUB3IvootYAY2WnOmIdgPGfqSI6tufQNnY= -github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= -github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/VictoriaMetrics/easyproto v0.1.4 h1:r8cNvo8o6sR4QShBXQd1bKw/VVLSQma/V2KhTBPf+Sc= github.com/VictoriaMetrics/easyproto v0.1.4/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= -github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= -github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= -github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb h1:f0BMgIjhZy4lSRHCXFbQst85f5agZAjtDMixQqBWNpc= github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= -github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= -github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= -github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= -github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= -github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= -github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBdAP4= -github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= -github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= -github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= -github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= -github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= -github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= -github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= -github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= -github.com/containerd/containerd v1.6.2 h1:pcaPUGbYW8kBw6OgIZwIVIeEhdWVrBzsoCfVJ5BjrLU= -github.com/containerd/containerd v1.6.2/go.mod h1:sidY30/InSE1j2vdD1ihtKoJz+lWdaXMdiAeIupaf+s= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= -github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= -github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= -github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= -github.com/containerd/continuity v0.2.2 h1:QSqfxcn8c+12slxwu00AtzXrsami0MJb/MQs9lOLHLA= -github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= -github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= -github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= -github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= -github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= -github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= -github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= -github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= -github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= -github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= -github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= -github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= -github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= -github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= -github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= -github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= -github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= -github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= -github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= -github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= -github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= -github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= -github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.14+incompatible h1:+T9/PRYWNDo5SZl5qS1r9Mo/0Q8AwxKKPtu9S1yxM0w= -github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fasthttp/router v1.4.1 h1:3xPUO+hy/HAkgGDSd5sX5w18cyGDIFbC7vip8KwPDk8= github.com/fasthttp/router v1.4.1/go.mod h1:4P0Kq4C882tA2evBKDW7De7hGfWmvV8FN+zqt8Lu49Q= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= -github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= -github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= -github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -413,7 +158,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -428,8 +172,6 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -443,15 +185,11 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -466,42 +204,17 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -513,116 +226,58 @@ github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXei github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= -github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= -github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= -github.com/moby/sys/mount v0.3.2 h1:uq/CiGDZPvr+c85RYHtKIUORFbmavBUyWH3E1NEyjqI= -github.com/moby/sys/mount v0.3.2/go.mod h1:iN27Ec0LtJ0Mx/++rE6t6mTdbbEEZd+oKfAHP1y6vHs= -github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/mountinfo v0.6.1 h1:+H/KnGEAGRpTrEAqNVQ2AM3SiwMgJUt/TXj+Z8cmCIc= -github.com/moby/sys/mountinfo v0.6.1/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= @@ -640,12 +295,6 @@ github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsC github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc= github.com/nspcc-dev/neo-go v0.106.2 h1:KXSJ2J5Oacc7LrX3r4jvnC8ihKqHs5NB21q4f2S3r9o= @@ -656,256 +305,127 @@ github.com/nspcc-dev/rfc6979 v0.2.1 h1:8wWxkamHWFmO790GsewSoKUSJjVnL1fmdRpokU/Rg github.com/nspcc-dev/rfc6979 v0.2.1/go.mod h1:Tk7h5kyUWkhjyO3zUgFFhy1v2vQv3BvQEntakdtqrWc= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= -github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= -github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= -github.com/opencontainers/runc v1.1.1 h1:PJ9DSs2sVwE0iVr++pAHE6QkS9tzcVWozlPifdwMgrU= -github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= -github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= -github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q= github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 h1:N3Af8f13ooDKcIhsmFT7Z05CStZWu4C7Md0uDEy4q6o= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873/go.mod h1:dmPawKuiAeG/aFYVs2i+Dyosoo7FNcm+Pi8iK6ZUrX8= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= -github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/ssgreg/journald v1.0.0 h1:0YmTDPJXxcWDPba12qNMdO6TxvfkFSYpFIJ31CwmLcU= github.com/ssgreg/journald v1.0.0/go.mod h1:RUckwmTM8ghGWPslq2+ZBZzbb9/2KgjzYZ4JEP+oRt0= -github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= -github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBNGN0TYb/7oKIPVn15JA= -github.com/testcontainers/testcontainers-go v0.13.0/go.mod h1:z1abufU633Eb/FmSBTzV6ntZAC1eZBYPtaFsn4nPuDk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/testcontainers/testcontainers-go v0.35.0 h1:uADsZpTKFAtp8SLK+hMwSaa+X+JiERHtd4sQAFmXeMo= +github.com/testcontainers/testcontainers-go v0.35.0/go.mod h1:oEVBj5zrfJTrgjwONs1SsRbnBtH9OKl+IGl3UMcr2B4= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 h1:GpfJ7OdNjS7BFTVwNCUI9L4aCJOFRbr5fdHqjdhoYE8= github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4/go.mod h1:f3jBhpWvuZmue0HZK52GzRHJOYHYSILs/c8+K2S/J+o= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= -github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= +github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA= github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= -github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= @@ -914,39 +434,26 @@ go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBq go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -982,31 +489,21 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1023,17 +520,12 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= @@ -1057,52 +549,29 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1112,51 +581,36 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1165,35 +619,25 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1216,17 +660,14 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1234,8 +675,6 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1244,7 +683,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1271,13 +709,11 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1286,7 +722,6 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -1297,17 +732,14 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1317,14 +749,10 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1: google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= -google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1334,12 +762,9 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1352,41 +777,20 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -1394,12 +798,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/gotestsum v1.7.0/go.mod h1:V1m4Jw3eBerhI/A6qCxUE07RnCg7ACkKj9BYcAm09V8= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1407,36 +807,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= -k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= -k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= -k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= -k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= -k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= -k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= @@ -1444,10 +814,3 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From a7617514d39c2bd06ae006a3e28a3471a6bcecb0 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 21 Jan 2025 12:59:25 +0300 Subject: [PATCH 504/548] [#193] Use selfhosted image registry instead of Docker Hub Existing AIO image tags referenced from our integration tests were manually synced to git.frostfs.info prior to this change. Signed-off-by: Vitaliy Potyarkin --- cmd/http-gw/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 0c2bdf4..c3c5de5 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -51,7 +51,7 @@ const ( func TestIntegration(t *testing.T) { rootCtx := context.Background() - aioImage := "truecloudlab/frostfs-aio:" + aioImage := "git.frostfs.info/truecloudlab/frostfs-aio:" versions := []string{ "1.2.7", "1.3.0", From 7901d00924e93f387766c1400195fb5a56c65abf Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 13 Dec 2024 16:00:31 +0300 Subject: [PATCH 505/548] [#170] Support tar.gz downloading Split DownloadZip handler on methods. Add handler DownloadTar for downloading tar.gz archives. Make methods more universal for using in both implementations Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 14 +- cmd/http-gw/settings.go | 15 +- config/config.env | 6 +- config/config.yaml | 10 +- docs/gate-configuration.md | 14 +- internal/handler/download.go | 228 +++++++++++++++++--------- internal/handler/handler.go | 4 +- internal/handler/handler_fuzz_test.go | 2 +- internal/handler/handler_test.go | 2 +- internal/handler/head.go | 2 +- internal/handler/utils.go | 7 + internal/logs/logs.go | 4 +- 12 files changed, 214 insertions(+), 94 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index e34386c..56c6be1 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -97,7 +97,7 @@ type ( mu sync.RWMutex defaultTimestamp bool - zipCompression bool + archiveCompression bool clientCut bool returnIndexPage bool indexPageTemplate string @@ -178,7 +178,7 @@ func (a *app) initAppSettings() { func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { defaultTimestamp := v.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) - zipCompression := v.GetBool(cfgZipCompression) + archiveCompression := fetchArchiveCompression(v) returnIndexPage := v.GetBool(cfgIndexPageEnabled) clientCut := v.GetBool(cfgClientCut) bufferMaxSizeForPut := v.GetUint64(cfgBufferMaxSizeForPut) @@ -197,7 +197,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { defer s.mu.Unlock() s.defaultTimestamp = defaultTimestamp - s.zipCompression = zipCompression + s.archiveCompression = archiveCompression s.returnIndexPage = returnIndexPage s.clientCut = clientCut s.bufferMaxSizeForPut = bufferMaxSizeForPut @@ -236,10 +236,10 @@ func (s *appSettings) DefaultTimestamp() bool { return s.defaultTimestamp } -func (s *appSettings) ZipCompression() bool { +func (s *appSettings) ArchiveCompression() bool { s.mu.RLock() defer s.mu.RUnlock() - return s.zipCompression + return s.archiveCompression } func (s *appSettings) IndexPageEnabled() bool { @@ -656,8 +656,10 @@ func (a *app) configureRouter(h *handler.Handler) { r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.HeadByAttribute)) r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZipped)) + r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZip)) r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) + r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadTar)) + r.OPTIONS("/tar/{cid}/{prefix:*}", a.addPreflight()) a.log.Info(logs.AddedPathZipCidPrefix) a.webServer.Handler = r.Handler diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 691e9ba..5cf06a0 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -128,8 +128,13 @@ const ( cfgResolveOrder = "resolve_order" // Zip compression. + // + // Deprecated: Use cfgArchiveCompression instead. cfgZipCompression = "zip.compression" + // Archive compression. + cfgArchiveCompression = "archive.compression" + // Runtime. cfgSoftMemoryLimit = "runtime.soft_memory_limit" @@ -255,9 +260,6 @@ func settings() *viper.Viper { // upload header v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) - // zip: - v.SetDefault(cfgZipCompression, false) - // metrics v.SetDefault(cfgPprofAddress, "localhost:8083") v.SetDefault(cfgPrometheusAddress, "localhost:8084") @@ -844,3 +846,10 @@ func fetchTracingAttributes(v *viper.Viper) (map[string]string, error) { return attributes, nil } + +func fetchArchiveCompression(v *viper.Viper) bool { + if v.IsSet(cfgZipCompression) { + return v.GetBool(cfgZipCompression) + } + return v.GetBool(cfgArchiveCompression) +} diff --git a/config/config.env b/config/config.env index 171889f..db619b5 100644 --- a/config/config.env +++ b/config/config.env @@ -97,9 +97,13 @@ HTTP_GW_REBALANCE_TIMER=30s # The number of errors on connection after which node is considered as unhealthy HTTP_GW_POOL_ERROR_THRESHOLD=100 -# Enable zip compression to download files by common prefix. +# Enable archive compression to download files by common prefix. +# DEPRECATED: Use HTTP_GW_ARCHIVE_COMPRESSION instead. HTTP_GW_ZIP_COMPRESSION=false +# Enable archive compression to download files by common prefix. +HTTP_GW_ARCHIVE_COMPRESSION=false + HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENDPOINT="localhost:4317" HTTP_GW_TRACING_EXPORTER="otlp_grpc" diff --git a/config/config.yaml b/config/config.yaml index eee84e5..a70ec9a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -116,13 +116,19 @@ pool_error_threshold: 100 # The number of errors on connection after which node # Number of workers in handler's worker pool worker_pool_size: 1000 -# Enable index page to see objects list for specified container and prefix +# Enables index page to see objects list for specified container and prefix index_page: enabled: false template_path: internal/handler/templates/index.gotmpl +# Deprecated: Use archive.compression instead zip: - compression: false # Enable zip compression to download files by common prefix. + # Enables zip compression to download files by common prefix. + compression: false + +archive: + # Enables archive compression to download files by common prefix. + compression: false runtime: soft_memory_limit: 1gb diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index ce7c0c7..6aadd1f 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -218,9 +218,10 @@ upload_header: |-------------------------|--------|---------------|---------------|-------------------------------------------------------------| | `use_default_timestamp` | `bool` | yes | `false` | Create timestamp for object if it isn't provided by header. | - # `zip` section +> **_DEPRECATED:_** Use archive section instead + ```yaml zip: compression: false @@ -230,6 +231,17 @@ zip: |---------------|--------|---------------|---------------|--------------------------------------------------------------| | `compression` | `bool` | yes | `false` | Enable zip compression when download files by common prefix. | +# `archive` section + +```yaml +archive: + compression: false +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------|--------|---------------|---------------|------------------------------------------------------------------| +| `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. | + # `pprof` section diff --git a/internal/handler/download.go b/internal/handler/download.go index 8766f0c..684e3b8 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -1,20 +1,21 @@ package handler import ( + "archive/tar" "archive/zip" "bufio" + "compress/gzip" "context" "errors" "fmt" "io" - "net/http" "net/url" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -46,7 +47,7 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { return } - req := h.newRequest(c, log) + req := newRequest(c, log) var objID oid.ID if checkS3Err == nil && shouldDownload(oidParam, downloadParam) { @@ -62,13 +63,6 @@ func shouldDownload(oidParam string, downloadParam bool) bool { return !isDir(oidParam) || downloadParam } -func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) request { - return request{ - RequestCtx: ctx, - log: log, - } -} - // DownloadByAttribute handles attribute-based download requests. func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { h.byAttribute(c, h.receiveFile) @@ -90,13 +84,61 @@ func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op return h.frostfs.SearchObjects(ctx, prm) } -func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { +// DownloadZip handles zip by prefix requests. +func (h *Handler) DownloadZip(c *fasthttp.RequestCtx) { + scid, _ := c.UserValue("cid").(string) + + ctx := utils.GetContextFromRequest(c) + log := utils.GetReqLogOrDefault(ctx, h.log) + bktInfo, err := h.getBucketInfo(ctx, scid, log) + if err != nil { + logAndSendBucketError(c, log, err) + return + } + resSearch, err := h.searchObjectsByPrefix(c, log, bktInfo.CID) + if err != nil { + return + } + + c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") + c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") + + c.SetBodyStreamWriter(h.getZipResponseWriter(ctx, log, resSearch, bktInfo)) +} + +func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { + return func(w *bufio.Writer) { + defer resSearch.Close() + + buf := make([]byte, 3<<20) + zipWriter := zip.NewWriter(w) + var objectsWritten int + + errIter := resSearch.Iterate(h.putObjectToArchive(ctx, log, bktInfo.CID, buf, + func(obj *object.Object) (io.Writer, error) { + objectsWritten++ + return h.createZipFile(zipWriter, obj) + }), + ) + if errIter != nil { + log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) + return + } else if objectsWritten == 0 { + log.Warn(logs.ObjectsNotFound) + } + if err := zipWriter.Close(); err != nil { + log.Error(logs.CloseZipWriter, zap.Error(err)) + } + } +} + +func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, error) { method := zip.Store if h.config.ZipCompression() { method = zip.Deflate } - filePath := getZipFilePath(obj) + filePath := getFilePath(obj) if len(filePath) == 0 || filePath[len(filePath)-1] == '/' { return nil, fmt.Errorf("invalid filepath '%s'", filePath) } @@ -108,99 +150,139 @@ func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, }) } -// DownloadZipped handles zip by prefix requests. -func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { +// DownloadTar forms tar.gz from objects by prefix. +func (h *Handler) DownloadTar(c *fasthttp.RequestCtx) { scid, _ := c.UserValue("cid").(string) - prefix, _ := c.UserValue("prefix").(string) ctx := utils.GetContextFromRequest(c) log := utils.GetReqLogOrDefault(ctx, h.log) - - prefix, err := url.QueryUnescape(prefix) - if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) - ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) - return - } - - log = log.With(zap.String("cid", scid), zap.String("prefix", prefix)) - bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { logAndSendBucketError(c, log, err) return } - - resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) + resSearch, err := h.searchObjectsByPrefix(c, log, bktInfo.CID) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) - ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return } - c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") - c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") - c.Response.SetStatusCode(http.StatusOK) + c.Response.Header.Set(fasthttp.HeaderContentType, "application/gzip") + c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.tar.gz\"") - c.SetBodyStreamWriter(func(w *bufio.Writer) { + c.SetBodyStreamWriter(h.getTarResponseWriter(ctx, log, resSearch, bktInfo)) +} + +func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { + return func(w *bufio.Writer) { defer resSearch.Close() - zipWriter := zip.NewWriter(w) + compressionLevel := gzip.NoCompression + if h.config.ZipCompression() { + compressionLevel = gzip.DefaultCompression + } - var bufZip []byte - var addr oid.Address + // ignore error because it's not nil only if compressionLevel argument is invalid + gzipWriter, _ := gzip.NewWriterLevel(w, compressionLevel) + tarWriter := tar.NewWriter(gzipWriter) - empty := true - called := false - btoken := bearerToken(ctx) - addr.SetContainer(bktInfo.CID) - - errIter := resSearch.Iterate(func(id oid.ID) bool { - called = true - - if empty { - bufZip = make([]byte, 3<<20) // the same as for upload + defer func() { + if err := tarWriter.Close(); err != nil { + log.Error(logs.CloseTarWriter, zap.Error(err)) } - empty = false - - addr.SetObject(id) - if err = h.zipObject(ctx, zipWriter, addr, btoken, bufZip); err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.String("oid", id.EncodeToString()), zap.Error(err)) + if err := gzipWriter.Close(); err != nil { + log.Error(logs.CloseGzipWriter, zap.Error(err)) } + }() - return false - }) + var objectsWritten int + buf := make([]byte, 3<<20) // the same as for upload + + errIter := resSearch.Iterate(h.putObjectToArchive(ctx, log, bktInfo.CID, buf, + func(obj *object.Object) (io.Writer, error) { + objectsWritten++ + return h.createTarFile(tarWriter, obj) + }), + ) if errIter != nil { log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) - } else if !called { - log.Error(logs.ObjectsNotFound) + } else if objectsWritten == 0 { + log.Warn(logs.ObjectsNotFound) } + } +} - if err = zipWriter.Close(); err != nil { - log.Error(logs.CloseZipWriter, zap.Error(err)) - } +func (h *Handler) createTarFile(tw *tar.Writer, obj *object.Object) (io.Writer, error) { + filePath := getFilePath(obj) + if len(filePath) == 0 || filePath[len(filePath)-1] == '/' { + return nil, fmt.Errorf("invalid filepath '%s'", filePath) + } + + return tw, tw.WriteHeader(&tar.Header{ + Name: filePath, + Mode: 0655, + Size: int64(obj.PayloadSize()), }) } -func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error { - prm := PrmObjectGet{ - PrmAuth: PrmAuth{ - BearerToken: btoken, - }, - Address: addr, - } +func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID cid.ID, buf []byte, createArchiveHeader func(obj *object.Object) (io.Writer, error)) func(id oid.ID) bool { + return func(id oid.ID) bool { + log = log.With(zap.String("oid", id.EncodeToString())) - resGet, err := h.frostfs.GetObject(ctx, prm) + prm := PrmObjectGet{ + PrmAuth: PrmAuth{ + BearerToken: bearerToken(ctx), + }, + Address: newAddress(cnrID, id), + } + + resGet, err := h.frostfs.GetObject(ctx, prm) + if err != nil { + log.Error(logs.FailedToGetObject, zap.Error(err)) + return false + } + + fileWriter, err := createArchiveHeader(&resGet.Header) + if err != nil { + log.Error(logs.FailedToAddObjectToArchive, zap.Error(err)) + return false + } + + if err = writeToArchive(resGet, fileWriter, buf); err != nil { + log.Error(logs.FailedToAddObjectToArchive, zap.Error(err)) + return false + } + + return false + } +} + +func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, cnrID cid.ID) (ResObjectSearch, error) { + scid := cnrID.EncodeToString() + prefix, _ := c.UserValue("prefix").(string) + + ctx := utils.GetContextFromRequest(c) + + prefix, err := url.QueryUnescape(prefix) if err != nil { - return fmt.Errorf("get FrostFS object: %v", err) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) + ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) + return nil, err } - objWriter, err := h.addObjectToZip(zipWriter, &resGet.Header) + log = log.With(zap.String("cid", scid), zap.String("prefix", prefix)) + + resSearch, err := h.search(ctx, cnrID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { - return fmt.Errorf("zip create header: %v", err) + log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) + return nil, err } + return resSearch, nil +} - if _, err = io.CopyBuffer(objWriter, resGet.Payload, bufZip); err != nil { +func writeToArchive(resGet *Object, objWriter io.Writer, buf []byte) error { + var err error + if _, err = io.CopyBuffer(objWriter, resGet.Payload, buf); err != nil { return fmt.Errorf("copy object payload to zip file: %v", err) } @@ -208,14 +290,10 @@ func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid return fmt.Errorf("object body close error: %w", err) } - if err = zipWriter.Flush(); err != nil { - return fmt.Errorf("flush zip writer: %v", err) - } - return nil } -func getZipFilePath(obj *object.Object) string { +func getFilePath(obj *object.Object) string { for _, attr := range obj.Attributes() { if attr.Key() == object.AttributeFilePath { return attr.Value() diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2f1c6ad..3d2b95d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -216,7 +216,7 @@ func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path } addr := newAddress(cnrID, foundOID.OID) - handler(ctx, h.newRequest(c, log), addr) + handler(ctx, newRequest(c, log), addr) } // byAttribute is a wrapper similar to byNativeAddress. @@ -265,7 +265,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte addr.SetContainer(bktInfo.CID) addr.SetObject(objID) - handler(ctx, h.newRequest(c, log), addr) + handler(ctx, newRequest(c, log), addr) } func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { diff --git a/internal/handler/handler_fuzz_test.go b/internal/handler/handler_fuzz_test.go index ad2ae6e..d71e8b0 100644 --- a/internal/handler/handler_fuzz_test.go +++ b/internal/handler/handler_fuzz_test.go @@ -517,7 +517,7 @@ func DoFuzzDownloadZipped(input []byte) int { r.SetUserValue("cid", cid) r.SetUserValue("prefix", prefix) - hc.Handler().DownloadZipped(r) + hc.Handler().DownloadZip(r) return fuzzSuccessExitCode } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index e1bc010..4784708 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -250,7 +250,7 @@ func TestBasic(t *testing.T) { t.Run("zip", func(t *testing.T) { r = prepareGetZipped(ctx, bktName, "") - hc.Handler().DownloadZipped(r) + hc.Handler().DownloadZip(r) readerAt := bytes.NewReader(r.Response.Body()) zipReader, err := zip.NewReader(readerAt, int64(len(r.Response.Body()))) diff --git a/internal/handler/head.go b/internal/handler/head.go index da96eff..94f5ccb 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -135,7 +135,7 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { return } - req := h.newRequest(c, log) + req := newRequest(c, log) var objID oid.ID if checkS3Err == nil { diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 7fdd396..971c3c8 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -24,6 +24,13 @@ type request struct { log *zap.Logger } +func newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) request { + return request{ + RequestCtx: ctx, + log: log, + } +} + func (r *request) handleFrostFSErr(err error, start time.Time) { logFields := []zap.Field{ zap.Stringer("elapsed", time.Since(start)), diff --git a/internal/logs/logs.go b/internal/logs/logs.go index f9b13b1..a4f206b 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -11,9 +11,12 @@ const ( ObjectNotFound = "object not found" // Error in ../../downloader/download.go ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go + FailedToGetObject = "failed to get object" // Error in ../../downloader/download.go IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go + CloseGzipWriter = "close gzip writer" // Error in ../../downloader/download.go + CloseTarWriter = "close tar writer" // Error in ../../downloader/download.go ServiceIsRunning = "service is running" // Info in ../../metrics/service.go ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go @@ -24,7 +27,6 @@ const ( IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go - CouldNotProcessHeaders = "could not process headers" // Error in ../../uploader/upload.go CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go CouldNotEncodeResponse = "could not encode response" // Error in ../../uploader/upload.go From 1e7309684bb41ba096563ed8d7c89a4e0fb33148 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 6 Dec 2024 15:01:16 +0300 Subject: [PATCH 506/548] [#170] Support .tar/.tgz unpacking during upload During upload if X-Explode-Archive is set, gate tries to read archive and create an object for each file. Each object acquires a FilePath attribute which is calculated relative to the archive root. Archive could have compression via Gzip if "Content-Encoding: gzip" header is specified Signed-off-by: Nikita Zinkevich --- internal/handler/download.go | 2 +- internal/handler/multipart.go | 4 +- internal/handler/upload.go | 245 +++++++++++++++++++++++----------- internal/handler/utils.go | 7 + internal/logs/logs.go | 19 ++- 5 files changed, 185 insertions(+), 92 deletions(-) diff --git a/internal/handler/download.go b/internal/handler/download.go index 684e3b8..4641052 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -257,7 +257,7 @@ func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID } func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, cnrID cid.ID) (ResObjectSearch, error) { - scid := cnrID.EncodeToString() + scid, _ := c.UserValue("cid").(string) prefix, _ := c.UserValue("prefix").(string) ctx := utils.GetContextFromRequest(c) diff --git a/internal/handler/multipart.go b/internal/handler/multipart.go index 213286c..ebf5edd 100644 --- a/internal/handler/multipart.go +++ b/internal/handler/multipart.go @@ -42,7 +42,9 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF // ignore multipart/form-data values if filename == "" { l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name)) - + if err = part.Close(); err != nil { + l.Warn(logs.FailedToCloseReader, zap.Error(err)) + } continue } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 9493635..d1953c9 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -1,13 +1,19 @@ package handler import ( + "archive/tar" + "bytes" + "compress/gzip" "context" "encoding/json" + "errors" "io" "net/http" + "path/filepath" "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" @@ -19,8 +25,9 @@ import ( ) const ( - jsonHeader = "application/json; charset=UTF-8" - drainBufSize = 4096 + jsonHeader = "application/json; charset=UTF-8" + drainBufSize = 4096 + explodeArchiveHeader = "X-Explode-Archive" ) type putResponse struct { @@ -43,11 +50,7 @@ func (pr *putResponse) encode(w io.Writer) error { // Upload handles multipart upload request. func (h *Handler) Upload(c *fasthttp.RequestCtx) { - var ( - file MultipartFile - idObj oid.ID - addr oid.Address - ) + var file MultipartFile scid, _ := c.UserValue("cid").(string) bodyStream := c.RequestBodyStream() @@ -63,20 +66,6 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { return } - defer func() { - // If the temporary reader can be closed - let's close it. - if file == nil { - return - } - err := file.Close() - log.Debug( - logs.CloseTemporaryMultipartFormFile, - zap.Stringer("address", addr), - zap.String("filename", file.FileName()), - zap.Error(err), - ) - }() - boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) @@ -86,53 +75,69 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { filtered, err := filterHeaders(log, &c.Request.Header) if err != nil { - log.Error(logs.CouldNotProcessHeaders, zap.Error(err)) + log.Error(logs.FailedToFilterHeaders, zap.Error(err)) ResponseError(c, err.Error(), fasthttp.StatusBadRequest) return } - now := time.Now() - if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { - if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { - log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err)) - } else { - now = parsed - } + if c.Request.Header.Peek(explodeArchiveHeader) != nil { + h.explodeArchive(request{c, log}, bktInfo, file, filtered) + } else { + h.uploadSingleObject(request{c, log}, bktInfo, file, filtered) } - if err = utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { - log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) - ResponseError(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) + // Multipart is multipart and thus can contain more than one part which + // we ignore at the moment. Also, when dealing with chunked encoding + // the last zero-length chunk might be left unread (because multipart + // reader only cares about its boundary and doesn't look further) and + // it will be (erroneously) interpreted as the start of the next + // pipelined header. Thus, we need to drain the body buffer. + for { + _, err = bodyStream.Read(drainBuf) + if err == io.EOF || errors.Is(err, io.ErrUnexpectedEOF) { + break + } + } +} + +func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file MultipartFile, filtered map[string]string) { + c, log := req.RequestCtx, req.log + setIfNotExist(filtered, object.AttributeFileName, file.FileName()) + + attributes, err := h.extractAttributes(c, log, filtered) + if err != nil { + log.Error(logs.FailedToGetAttributes, zap.Error(err)) + ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) return } - attributes := make([]object.Attribute, 0, len(filtered)) - // prepares attributes from filtered headers - for key, val := range filtered { - attribute := object.NewAttribute() - attribute.SetKey(key) - attribute.SetValue(val) - attributes = append(attributes, *attribute) + idObj, err := h.uploadObject(c, bkt, attributes, file) + if err != nil { + h.handlePutFrostFSErr(c, err, log) + return } - // sets FileName attribute if it wasn't set from header - if _, ok := filtered[object.AttributeFileName]; !ok { - filename := object.NewAttribute() - filename.SetKey(object.AttributeFileName) - filename.SetValue(file.FileName()) - attributes = append(attributes, *filename) - } - // sets Timestamp attribute if it wasn't set from header and enabled by settings - if _, ok := filtered[object.AttributeTimestamp]; !ok && h.config.DefaultTimestamp() { - timestamp := object.NewAttribute() - timestamp.SetKey(object.AttributeTimestamp) - timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) - attributes = append(attributes, *timestamp) + log.Debug(logs.ObjectUploaded, + zap.String("oid", idObj.EncodeToString()), + zap.String("FileName", file.FileName()), + ) + + addr := newAddress(bkt.CID, idObj) + c.Response.Header.SetContentType(jsonHeader) + // Try to return the response, otherwise, if something went wrong, throw an error. + if err = newPutResponse(addr).encode(c); err != nil { + log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) + ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) + return } +} + +func (h *Handler) uploadObject(c *fasthttp.RequestCtx, bkt *data.BucketInfo, attrs []object.Attribute, file io.Reader) (oid.ID, error) { + ctx := utils.GetContextFromRequest(c) obj := object.New() - obj.SetContainerID(bktInfo.CID) + obj.SetContainerID(bkt.CID) obj.SetOwnerID(*h.ownerID) - obj.SetAttributes(attributes...) + obj.SetAttributes(attrs...) prm := PrmObjectCreate{ PrmAuth: PrmAuth{ @@ -141,40 +146,120 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { Object: obj, Payload: file, ClientCut: h.config.ClientCut(), - WithoutHomomorphicHash: bktInfo.HomomorphicHashDisabled, + WithoutHomomorphicHash: bkt.HomomorphicHashDisabled, BufferMaxSize: h.config.BufferMaxSizeForPut(), } - if idObj, err = h.frostfs.CreateObject(ctx, prm); err != nil { - h.handlePutFrostFSErr(c, err, log) - return + idObj, err := h.frostfs.CreateObject(ctx, prm) + if err != nil { + return oid.ID{}, err } - addr.SetObject(idObj) - addr.SetContainer(bktInfo.CID) + return idObj, nil +} - // Try to return the response, otherwise, if something went wrong, throw an error. - if err = newPutResponse(addr).encode(c); err != nil { - log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) - ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) - - return - } - // Multipart is multipart and thus can contain more than one part which - // we ignore at the moment. Also, when dealing with chunked encoding - // the last zero-length chunk might be left unread (because multipart - // reader only cares about its boundary and doesn't look further) and - // it will be (erroneously) interpreted as the start of the next - // pipelined header. Thus we need to drain the body buffer. - for { - _, err = bodyStream.Read(drainBuf) - if err == io.EOF || err == io.ErrUnexpectedEOF { - break +func (h *Handler) extractAttributes(c *fasthttp.RequestCtx, log *zap.Logger, filtered map[string]string) ([]object.Attribute, error) { + now := time.Now() + if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { + if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { + log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err)) + } else { + now = parsed } } - // Report status code and content type. - c.Response.SetStatusCode(fasthttp.StatusOK) - c.Response.Header.SetContentType(jsonHeader) + if err := utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { + log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) + return nil, err + } + attributes := make([]object.Attribute, 0, len(filtered)) + // prepares attributes from filtered headers + for key, val := range filtered { + attribute := newAttribute(key, val) + attributes = append(attributes, attribute) + } + // sets Timestamp attribute if it wasn't set from header and enabled by settings + if _, ok := filtered[object.AttributeTimestamp]; !ok && h.config.DefaultTimestamp() { + timestamp := newAttribute(object.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10)) + attributes = append(attributes, timestamp) + } + + return attributes, nil +} + +func newAttribute(key string, val string) object.Attribute { + attr := object.NewAttribute() + attr.SetKey(key) + attr.SetValue(val) + return *attr +} + +// explodeArchive read files from archive and creates objects for each of them. +// Sets FilePath attribute with name from tar.Header. +func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.ReadCloser, filtered map[string]string) { + c, log := req.RequestCtx, req.log + + // remove user attributes which vary for each file in archive + // to guarantee that they won't appear twice + delete(filtered, object.AttributeFileName) + delete(filtered, object.AttributeFilePath) + + commonAttributes, err := h.extractAttributes(c, log, filtered) + if err != nil { + log.Error(logs.FailedToGetAttributes, zap.Error(err)) + ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) + return + } + attributes := commonAttributes + + reader := file + if bytes.EqualFold(c.Request.Header.Peek(fasthttp.HeaderContentEncoding), []byte("gzip")) { + log.Debug(logs.GzipReaderSelected) + gzipReader, err := gzip.NewReader(file) + if err != nil { + log.Error(logs.FailedToCreateGzipReader, zap.Error(err)) + ResponseError(c, "could read gzip file: "+err.Error(), fasthttp.StatusBadRequest) + return + } + defer func() { + if err := gzipReader.Close(); err != nil { + log.Warn(logs.FailedToCloseReader, zap.Error(err)) + } + }() + reader = gzipReader + } + + tarReader := tar.NewReader(reader) + for { + obj, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } else if err != nil { + log.Error(logs.FailedToReadFileFromTar, zap.Error(err)) + ResponseError(c, "could not get next entry: "+err.Error(), fasthttp.StatusBadRequest) + return + } + + if isDir(obj.Name) { + continue + } + + // set varying attributes + attributes = attributes[:len(commonAttributes)] + fileName := filepath.Base(obj.Name) + attributes = append(attributes, newAttribute(object.AttributeFilePath, obj.Name)) + attributes = append(attributes, newAttribute(object.AttributeFileName, fileName)) + + idObj, err := h.uploadObject(c, bkt, attributes, tarReader) + if err != nil { + h.handlePutFrostFSErr(c, err, log) + return + } + + log.Debug(logs.ObjectUploaded, + zap.String("oid", idObj.EncodeToString()), + zap.String("FileName", fileName), + ) + } } func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *zap.Logger) { diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 971c3c8..74932f3 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -101,6 +101,13 @@ func newAddress(cnr cid.ID, obj oid.ID) oid.Address { return addr } +// setIfNotExist sets key value to map if key is not present yet. +func setIfNotExist(m map[string]string, key, value string) { + if _, ok := m[key]; !ok { + m[key] = value + } +} + func ResponseError(r *fasthttp.RequestCtx, msg string, code int) { r.Error(msg+"\n", code) } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index a4f206b..68270ed 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -4,8 +4,6 @@ const ( CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/* CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go - WrongObjectID = "wrong object id" // Error in ../../downloader/download.go - GetLatestObjectVersion = "get latest object version" // Error in ../../downloader/download.go ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go ObjectNotFound = "object not found" // Error in ../../downloader/download.go @@ -15,8 +13,6 @@ const ( IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go - CloseGzipWriter = "close gzip writer" // Error in ../../downloader/download.go - CloseTarWriter = "close tar writer" // Error in ../../downloader/download.go ServiceIsRunning = "service is running" // Info in ../../metrics/service.go ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go @@ -25,7 +21,6 @@ const ( CantGracefullyShutDownService = "can't gracefully shut down service, force stop" // Error in ../../metrics/service.go IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go - CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go @@ -81,11 +76,6 @@ const ( InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go FailedToUnescapeQuery = "failed to unescape query" - FailedToParseAddressInTreeNode = "failed to parse object addr in tree node" - SettingsNodeInvalidOwnerKey = "settings node: invalid owner key" - SystemNodeHasMultipleIDs = "system node has multiple ids" - FailedToRemoveOldSystemNode = "failed to remove old system node" - BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids" ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" @@ -96,4 +86,13 @@ const ( MultinetConfigWontBeUpdated = "multinet config won't be updated" ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" CouldntCacheNetmap = "couldn't cache netmap" + FailedToFilterHeaders = "failed to filter headers" + FailedToReadFileFromTar = "failed to read file from tar" + FailedToGetAttributes = "failed to get attributes" + ObjectUploaded = "object uploaded" + CloseGzipWriter = "close gzip writer" + CloseTarWriter = "close tar writer" + FailedToCloseReader = "failed to close reader" + FailedToCreateGzipReader = "failed to create gzip reader" + GzipReaderSelected = "gzip reader selected" ) From 1e897aa3c307df14bdbec1d5b690e8c187a88544 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 6 Dec 2024 15:01:16 +0300 Subject: [PATCH 507/548] [#170] Updated docs and configuration of archive section Signed-off-by: Nikita Zinkevich --- docs/api.md | 37 +++++++++++++++++--------------- internal/handler/download.go | 4 ++-- internal/handler/handler.go | 2 +- internal/handler/handler_test.go | 2 +- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/docs/api.md b/docs/api.md index e59956a..d099915 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,11 +1,11 @@ # HTTP Gateway Specification -| Route | Description | -|-------------------------------------------------|----------------------------------------------| -| `/upload/{cid}` | [Put object](#put-object) | -| `/get/{cid}/{oid}` | [Get object](#get-object) | -| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) | -| `/zip/{cid}/{prefix}` | [Download objects in archive](#download-zip) | +| Route | Description | +|-------------------------------------------------|--------------------------------------------------| +| `/upload/{cid}` | [Put object](#put-object) | +| `/get/{cid}/{oid}` | [Get object](#get-object) | +| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) | +| `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}` | [Download objects in archive](#download-archive) | **Note:** `cid` parameter can be base58 encoded container ID or container name (the name must be registered in NNS, see appropriate section in [nns.md](./nns.md)). @@ -56,12 +56,14 @@ Upload file as object with attributes to FrostFS. ###### Headers -| Header | Description | -|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| -| Common headers | See [bearer token](#bearer-token). | -| `X-Attribute-System-*` | Used to set system FrostFS object attributes
(e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). | -| `X-Attribute-*` | Used to set regular object attributes
(e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | -| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | +| Header | Description | +|------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Common headers | See [bearer token](#bearer-token). | +| `X-Attribute-System-*` | Used to set system FrostFS object attributes
(e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). | +| `X-Attribute-*` | Used to set regular object attributes
(e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | +| `X-Explode-Archive` | If set, gate tries to read files from uploading `tar` archive and creates an object for each file in it. Uploading `tar` could be compressed via Gzip by setting a `Content-Encoding` header. Sets a `FilePath` attribute as a relative path from archive root and a `FileName` as the last path element of the `FilePath`. | +| `Content-Encoding` | If set and value is `gzip`, gate will handle uploading file as a `Gzip` compressed `tar` file. | +| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority): @@ -269,9 +271,9 @@ If more than one object is found, an arbitrary one will be used to get attribute | 400 | Some error occurred during operation. | | 404 | Container or object not found. | -## Download zip +## Download archive -Route: `/zip/{cid}/{prefix}` +Route: `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}` | Route parameter | Type | Description | |-----------------|-----------|---------------------------------------------------------| @@ -282,12 +284,13 @@ Route: `/zip/{cid}/{prefix}` #### GET -Find objects by prefix for `FilePath` attributes. Return found objects in zip archive. +Find objects by prefix for `FilePath` attributes. Return found objects in zip or tar archive. Name of files in archive sets to `FilePath` attribute of objects. Time of files sets to time when object has started downloading. -You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` route. +You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` or +`/tar/{cid}/` route. -Archive can be compressed (see http-gw [configuration](gate-configuration.md#zip-section)). +Archive can be compressed (see http-gw [configuration](gate-configuration.md#archive-section)). ##### Request diff --git a/internal/handler/download.go b/internal/handler/download.go index 4641052..d5fac23 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -134,7 +134,7 @@ func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, res func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, error) { method := zip.Store - if h.config.ZipCompression() { + if h.config.ArchiveCompression() { method = zip.Deflate } @@ -177,7 +177,7 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res defer resSearch.Close() compressionLevel := gzip.NoCompression - if h.config.ZipCompression() { + if h.config.ArchiveCompression() { compressionLevel = gzip.DefaultCompression } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 3d2b95d..4e23c3b 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -28,7 +28,7 @@ import ( type Config interface { DefaultTimestamp() bool - ZipCompression() bool + ArchiveCompression() bool ClientCut() bool IndexPageEnabled() bool IndexPageTemplate() string diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 4784708..53c9739 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -66,7 +66,7 @@ func (c *configMock) DefaultTimestamp() bool { return false } -func (c *configMock) ZipCompression() bool { +func (c *configMock) ArchiveCompression() bool { return false } From 36bd3e2d43e1615fc7fed32e8874fd9db5eb8562 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Wed, 25 Dec 2024 13:08:22 +0300 Subject: [PATCH 508/548] [#170] logs: Remove comments Signed-off-by: Nikita Zinkevich --- internal/logs/logs.go | 148 +++++++++++++++++++++--------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 68270ed..5f60b9b 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -1,80 +1,80 @@ package logs const ( - CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/* - CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go - CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go - ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go - CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go - ObjectNotFound = "object not found" // Error in ../../downloader/download.go - ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go - FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go - FailedToGetObject = "failed to get object" // Error in ../../downloader/download.go - IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go - ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go - CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go - ServiceIsRunning = "service is running" // Info in ../../metrics/service.go - ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go - ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go - ShuttingDownService = "shutting down service" // Info in ../../metrics/service.go - CantShutDownService = "can't shut down service" // Panic in ../../metrics/service.go - CantGracefullyShutDownService = "can't gracefully shut down service, force stop" // Error in ../../metrics/service.go - IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go - IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go - CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go - CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go - CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go - CouldNotEncodeResponse = "could not encode response" // Error in ../../uploader/upload.go - CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go - AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go - FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go - FailedToCreateWorkerPool = "failed to create worker pool" // Fatal in ../../app.go - FailedToReadIndexPageTemplate = "failed to read index page template" // Error in ../../app.go - SetCustomIndexPageTemplate = "set custom index page template" // Info in ../../app.go - ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go - MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go - NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" // Info in ../../app.go - StartingApplication = "starting application" // Info in ../../app.go - StartingServer = "starting server" // Info in ../../app.go - ListenAndServe = "listen and serve" // Fatal in ../../app.go - ShuttingDownWebServer = "shutting down web server" // Info in ../../app.go - FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../app.go - SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../app.go - FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../app.go - FailedToReloadConfig = "failed to reload config" // Warn in ../../app.go - LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../app.go - FailedToUpdateResolvers = "failed to update resolvers" // Warn in ../../app.go - FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../app.go - SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../app.go - AddedPathUploadCid = "added path /upload/{cid}" // Info in ../../app.go - AddedPathGetCidOid = "added path /get/{cid}/{oid}" // Info in ../../app.go - AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" // Info in ../../app.go - AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" // Info in ../../app.go - Request = "request" // Info in ../../app.go - CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" // Error in ../../app.go - FailedToAddServer = "failed to add server" // Warn in ../../app.go - AddServer = "add server" // Info in ../../app.go - NoHealthyServers = "no healthy servers" // Fatal in ../../app.go - FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../app.go - TracingConfigUpdated = "tracing config updated" // Info in ../../app.go - ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" // Warn in ../../app.go - RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" // Warn in ../../app.go - RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" // Info in ../../app.go - CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../settings.go - UsingCredentials = "using credentials" // Info in ../../settings.go - FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../settings.go - FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../settings.go - FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../settings.go - FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../settings.go - AddedStoragePeer = "added storage peer" // Info in ../../settings.go - CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go - CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go - FailedToSumbitTaskToPool = "failed to submit task to pool" // Error in ../handler/browse.go - FailedToHeadObject = "failed to head object" // Error in ../handler/browse.go - FailedToIterateOverResponse = "failed to iterate over search response" // Error in ../handler/browse.go - InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go - InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go - InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go + CouldntParseCreationDate = "couldn't parse creation date" + CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" + CouldNotReceiveObject = "could not receive object" + ObjectWasDeleted = "object was deleted" + CouldNotSearchForObjects = "could not search for objects" + ObjectNotFound = "object not found" + ReadObjectListFailed = "read object list failed" + FailedToAddObjectToArchive = "failed to add object to archive" + FailedToGetObject = "failed to get object" + IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" + ObjectsNotFound = "objects not found" + CloseZipWriter = "close zip writer" + ServiceIsRunning = "service is running" + ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" + ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" + ShuttingDownService = "shutting down service" + CantShutDownService = "can't shut down service" + CantGracefullyShutDownService = "can't gracefully shut down service, force stop" + IgnorePartEmptyFormName = "ignore part, empty form name" + IgnorePartEmptyFilename = "ignore part, empty filename" + CouldNotReceiveMultipartForm = "could not receive multipart/form" + CouldNotParseClientTime = "could not parse client time" + CouldNotPrepareExpirationHeader = "could not prepare expiration header" + CouldNotEncodeResponse = "could not encode response" + CouldNotStoreFileInFrostfs = "could not store file in frostfs" + AddAttributeToResultObject = "add attribute to result object" + FailedToCreateResolver = "failed to create resolver" + FailedToCreateWorkerPool = "failed to create worker pool" + FailedToReadIndexPageTemplate = "failed to read index page template" + SetCustomIndexPageTemplate = "set custom index page template" + ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" + MetricsAreDisabled = "metrics are disabled" + NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" + StartingApplication = "starting application" + StartingServer = "starting server" + ListenAndServe = "listen and serve" + ShuttingDownWebServer = "shutting down web server" + FailedToShutdownTracing = "failed to shutdown tracing" + SIGHUPConfigReloadStarted = "SIGHUP config reload started" + FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" + FailedToReloadConfig = "failed to reload config" + LogLevelWontBeUpdated = "log level won't be updated" + FailedToUpdateResolvers = "failed to update resolvers" + FailedToReloadServerParameters = "failed to reload server parameters" + SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" + AddedPathUploadCid = "added path /upload/{cid}" + AddedPathGetCidOid = "added path /get/{cid}/{oid}" + AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" + AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" + Request = "request" + CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" + FailedToAddServer = "failed to add server" + AddServer = "add server" + NoHealthyServers = "no healthy servers" + FailedToInitializeTracing = "failed to initialize tracing" + TracingConfigUpdated = "tracing config updated" + ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" + RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" + RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" + CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" + UsingCredentials = "using credentials" + FailedToCreateConnectionPool = "failed to create connection pool" + FailedToDialConnectionPool = "failed to dial connection pool" + FailedToCreateTreePool = "failed to create tree pool" + FailedToDialTreePool = "failed to dial tree pool" + AddedStoragePeer = "added storage peer" + CouldntGetBucket = "could not get bucket" + CouldntPutBucketIntoCache = "couldn't put bucket info into cache" + FailedToSumbitTaskToPool = "failed to submit task to pool" + FailedToHeadObject = "failed to head object" + FailedToIterateOverResponse = "failed to iterate over search response" + InvalidCacheEntryType = "invalid cache entry type" + InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" + InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" FailedToUnescapeQuery = "failed to unescape query" ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" From 87ace4f8f7eee2cc128aa274fa0b19be55e441c5 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 28 Jan 2025 17:43:11 +0300 Subject: [PATCH 509/548] [#201] govulncheck: Use patch release with security fixes https://go.dev/doc/devel/release#go1.23.minor Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/vulncheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 76e2965..529e8a8 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -16,7 +16,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.22.11' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest From 526da379ad64439ed5e4e92774c8c9e57ff81564 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 27 Jan 2025 12:46:26 +0300 Subject: [PATCH 510/548] [#199] Fix SIGHUP panic Signed-off-by: Marina Biryukova --- cmd/http-gw/app.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 56c6be1..35a46ba 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -130,6 +130,7 @@ func newApp(ctx context.Context, v *viper.Viper) App { a := &app{ ctx: ctx, log: log.logger, + logLevel: log.lvl, cfg: v, loggerSettings: logSettings, webServer: new(fasthttp.Server), From a6fdaf9456bad2799dcf36ee95731e45c32505c8 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 27 Jan 2025 13:16:48 +0300 Subject: [PATCH 511/548] [#199] Clear app services list Signed-off-by: Marina Biryukova --- cmd/http-gw/app.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 35a46ba..3df637b 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -616,6 +616,8 @@ func (a *app) configReload(ctx context.Context) { } func (a *app) startServices() { + a.services = a.services[:0] + pprofConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPprofEnabled), Address: a.cfg.GetString(cfgPprofAddress)} pprofService := metrics.NewPprofService(a.log, pprofConfig) a.services = append(a.services, pprofService) From 8de06e23a07552f0083a4f6bac54dea1bce485bd Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 27 Jan 2025 13:17:02 +0300 Subject: [PATCH 512/548] [#199] Use default value if config param is unset after SIGHUP Signed-off-by: Marina Biryukova --- cmd/http-gw/app.go | 72 ++++----- cmd/http-gw/integration_test.go | 17 +-- cmd/http-gw/main.go | 4 +- cmd/http-gw/settings.go | 253 ++++++++++++++++++++------------ cmd/http-gw/settings_test.go | 60 ++++++++ 5 files changed, 269 insertions(+), 137 deletions(-) create mode 100644 cmd/http-gw/settings_test.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 3df637b..ac20d29 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -56,7 +56,7 @@ type ( treePool *treepool.Pool key *keys.PrivateKey owner *user.ID - cfg *viper.Viper + cfg *appCfg webServer *fasthttp.Server webDone chan struct{} resolver *resolver.ContainerResolver @@ -123,35 +123,35 @@ type ( } ) -func newApp(ctx context.Context, v *viper.Viper) App { +func newApp(ctx context.Context, cfg *appCfg) App { logSettings := &loggerSettings{} - log := pickLogger(v, logSettings) + log := pickLogger(cfg.config(), logSettings) a := &app{ ctx: ctx, log: log.logger, logLevel: log.lvl, - cfg: v, + cfg: cfg, loggerSettings: logSettings, webServer: new(fasthttp.Server), webDone: make(chan struct{}), - bucketCache: cache.NewBucketCache(getBucketCacheOptions(v, log.logger), v.GetBool(cfgFeaturesTreePoolNetmapSupport)), + bucketCache: cache.NewBucketCache(getBucketCacheOptions(cfg.config(), log.logger), cfg.config().GetBool(cfgFeaturesTreePoolNetmapSupport)), } a.initAppSettings() // -- setup FastHTTP server -- a.webServer.Name = "frost-http-gw" - a.webServer.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize) - a.webServer.WriteBufferSize = a.cfg.GetInt(cfgWebWriteBufferSize) - a.webServer.ReadTimeout = a.cfg.GetDuration(cfgWebReadTimeout) - a.webServer.WriteTimeout = a.cfg.GetDuration(cfgWebWriteTimeout) + a.webServer.ReadBufferSize = a.config().GetInt(cfgWebReadBufferSize) + a.webServer.WriteBufferSize = a.config().GetInt(cfgWebWriteBufferSize) + a.webServer.ReadTimeout = a.config().GetDuration(cfgWebReadTimeout) + a.webServer.WriteTimeout = a.config().GetDuration(cfgWebWriteTimeout) a.webServer.DisableHeaderNamesNormalizing = true a.webServer.NoDefaultServerHeader = true a.webServer.NoDefaultContentType = true - a.webServer.MaxRequestBodySize = a.cfg.GetInt(cfgWebMaxRequestBodySize) + a.webServer.MaxRequestBodySize = a.config().GetInt(cfgWebMaxRequestBodySize) a.webServer.DisablePreParseMultipartForm = true - a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) + a.webServer.StreamRequestBody = a.config().GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- a.initPools(ctx) @@ -168,13 +168,17 @@ func newApp(ctx context.Context, v *viper.Viper) App { return a } +func (a *app) config() *viper.Viper { + return a.cfg.config() +} + func (a *app) initAppSettings() { a.settings = &appSettings{ - reconnectInterval: fetchReconnectInterval(a.cfg), - dialerSource: getDialerSource(a.log, a.cfg), - workerPoolSize: a.cfg.GetInt(cfgWorkerPoolSize), + reconnectInterval: fetchReconnectInterval(a.config()), + dialerSource: getDialerSource(a.log, a.config()), + workerPoolSize: a.config().GetInt(cfgWorkerPoolSize), } - a.settings.update(a.cfg, a.log) + a.settings.update(a.config(), a.log) } func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { @@ -327,11 +331,11 @@ func (a *app) initResolver() { func (a *app) getResolverConfig() ([]string, *resolver.Config) { resolveCfg := &resolver.Config{ FrostFS: frostfs.NewResolverFrostFS(a.pool), - RPCAddress: a.cfg.GetString(cfgRPCEndpoint), + RPCAddress: a.config().GetString(cfgRPCEndpoint), Settings: a.settings, } - order := a.cfg.GetStringSlice(cfgResolveOrder) + order := a.config().GetStringSlice(cfgResolveOrder) if resolveCfg.RPCAddress == "" { order = remove(order, resolver.NNSResolver) a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided) @@ -346,7 +350,7 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) { func (a *app) initMetrics() { gateMetricsProvider := metrics.NewGateMetrics(a.pool) - a.metrics = newGateMetrics(a.log, gateMetricsProvider, a.cfg.GetBool(cfgPrometheusEnabled)) + a.metrics = newGateMetrics(a.log, gateMetricsProvider, a.config().GetBool(cfgPrometheusEnabled)) a.metrics.SetHealth(metrics.HealthStatusStarting) a.loggerSettings.setMetrics(a.metrics.provider) } @@ -574,22 +578,22 @@ func (a *app) shutdownTracing() { func (a *app) configReload(ctx context.Context) { a.log.Info(logs.SIGHUPConfigReloadStarted) - if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) { + if !a.config().IsSet(cmdConfig) && !a.config().IsSet(cmdConfigDir) { a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed) return } - if err := readInConfig(a.cfg); err != nil { + if err := a.cfg.reload(); err != nil { a.log.Warn(logs.FailedToReloadConfig, zap.Error(err)) return } - if lvl, err := getLogLevel(a.cfg); err != nil { + if lvl, err := getLogLevel(a.config()); err != nil { a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err)) } else { a.logLevel.SetLevel(lvl) } - if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.cfg, a.log)); err != nil { + if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil { a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err)) } @@ -606,9 +610,9 @@ func (a *app) configReload(ctx context.Context) { a.stopServices() a.startServices() - a.settings.update(a.cfg, a.log) + a.settings.update(a.config(), a.log) - a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled)) + a.metrics.SetEnabled(a.config().GetBool(cfgPrometheusEnabled)) a.initTracing(ctx) a.setHealthStatus() @@ -618,12 +622,12 @@ func (a *app) configReload(ctx context.Context) { func (a *app) startServices() { a.services = a.services[:0] - pprofConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPprofEnabled), Address: a.cfg.GetString(cfgPprofAddress)} + pprofConfig := metrics.Config{Enabled: a.config().GetBool(cfgPprofEnabled), Address: a.config().GetString(cfgPprofAddress)} pprofService := metrics.NewPprofService(a.log, pprofConfig) a.services = append(a.services, pprofService) go pprofService.Start() - prometheusConfig := metrics.Config{Enabled: a.cfg.GetBool(cfgPrometheusEnabled), Address: a.cfg.GetString(cfgPrometheusAddress)} + prometheusConfig := metrics.Config{Enabled: a.config().GetBool(cfgPrometheusEnabled), Address: a.config().GetString(cfgPrometheusAddress)} prometheusService := metrics.NewPrometheusService(a.log, prometheusConfig) a.services = append(a.services, prometheusService) go prometheusService.Start() @@ -850,7 +854,7 @@ func (a *app) AppParams() *handler.AppParams { } func (a *app) initServers(ctx context.Context) { - serversInfo := fetchServers(a.cfg, a.log) + serversInfo := fetchServers(a.config(), a.log) a.servers = make([]Server, 0, len(serversInfo)) for _, serverInfo := range serversInfo { @@ -877,7 +881,7 @@ func (a *app) initServers(ctx context.Context) { } func (a *app) updateServers() error { - serversInfo := fetchServers(a.cfg, a.log) + serversInfo := fetchServers(a.config(), a.log) a.mu.Lock() defer a.mu.Unlock() @@ -935,15 +939,15 @@ func (a *app) initTracing(ctx context.Context) { instanceID = a.servers[0].Address() } cfg := tracing.Config{ - Enabled: a.cfg.GetBool(cfgTracingEnabled), - Exporter: tracing.Exporter(a.cfg.GetString(cfgTracingExporter)), - Endpoint: a.cfg.GetString(cfgTracingEndpoint), + Enabled: a.config().GetBool(cfgTracingEnabled), + Exporter: tracing.Exporter(a.config().GetString(cfgTracingExporter)), + Endpoint: a.config().GetString(cfgTracingEndpoint), Service: "frostfs-http-gw", InstanceID: instanceID, Version: Version, } - if trustedCa := a.cfg.GetString(cfgTracingTrustedCa); trustedCa != "" { + if trustedCa := a.config().GetString(cfgTracingTrustedCa); trustedCa != "" { caBytes, err := os.ReadFile(trustedCa) if err != nil { a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) @@ -958,7 +962,7 @@ func (a *app) initTracing(ctx context.Context) { cfg.ServerCaCertPool = certPool } - attributes, err := fetchTracingAttributes(a.cfg) + attributes, err := fetchTracingAttributes(a.config()) if err != nil { a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) return @@ -981,7 +985,7 @@ func (a *app) setRuntimeParameters() { return } - softMemoryLimit := fetchSoftMemoryLimit(a.cfg) + softMemoryLimit := fetchSoftMemoryLimit(a.config()) previous := debug.SetMemoryLimit(softMemoryLimit) if softMemoryLimit != previous { a.log.Info(logs.RuntimeSoftMemoryLimitUpdated, diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index c3c5de5..2596bee 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -32,7 +32,6 @@ import ( docker "github.com/docker/docker/api/types/container" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/wallet" - "github.com/spf13/viper" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" @@ -108,8 +107,8 @@ func runServer(pathToWallet string) (App, context.CancelFunc) { cancelCtx, cancel := context.WithCancel(context.Background()) v := getDefaultConfig() - v.Set(cfgWalletPath, pathToWallet) - v.Set(cfgWalletPassphrase, "") + v.config().Set(cfgWalletPath, pathToWallet) + v.config().Set(cfgWalletPassphrase, "") application := newApp(cancelCtx, v) go application.Serve() @@ -452,14 +451,14 @@ func createDockerContainer(ctx context.Context, t *testing.T, image string) test return aioC } -func getDefaultConfig() *viper.Viper { +func getDefaultConfig() *appCfg { v := settings() - v.SetDefault(cfgPeers+".0.address", "localhost:8080") - v.SetDefault(cfgPeers+".0.weight", 1) - v.SetDefault(cfgPeers+".0.priority", 1) + v.config().SetDefault(cfgPeers+".0.address", "localhost:8080") + v.config().SetDefault(cfgPeers+".0.weight", 1) + v.config().SetDefault(cfgPeers+".0.priority", 1) - v.SetDefault(cfgRPCEndpoint, "http://localhost:30333") - v.SetDefault("server.0.address", testListenAddress) + v.config().SetDefault(cfgRPCEndpoint, "http://localhost:30333") + v.config().SetDefault("server.0.address", testListenAddress) return v } diff --git a/cmd/http-gw/main.go b/cmd/http-gw/main.go index fdd148c..002f190 100644 --- a/cmd/http-gw/main.go +++ b/cmd/http-gw/main.go @@ -8,9 +8,9 @@ import ( func main() { globalContext, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - v := settings() + cfg := settings() - application := newApp(globalContext, v) + application := newApp(globalContext, cfg) go application.Serve() application.Wait() } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 5cf06a0..5670f87 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -12,6 +12,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" @@ -197,14 +198,72 @@ type Logger struct { lvl zap.AtomicLevel } -func settings() *viper.Viper { +type appCfg struct { + flags *pflag.FlagSet + + mu sync.RWMutex + settings *viper.Viper +} + +func (a *appCfg) reload() error { + old := a.config() + + v, err := newViper(a.flags) + if err != nil { + return err + } + + if old.IsSet(cmdConfig) { + v.Set(cmdConfig, old.Get(cmdConfig)) + } + if old.IsSet(cmdConfigDir) { + v.Set(cmdConfigDir, old.Get(cmdConfigDir)) + } + + if err = readInConfig(v); err != nil { + return err + } + + a.setConfig(v) + return nil +} + +func (a *appCfg) config() *viper.Viper { + a.mu.RLock() + defer a.mu.RUnlock() + + return a.settings +} + +func (a *appCfg) setConfig(v *viper.Viper) { + a.mu.Lock() + a.settings = v + a.mu.Unlock() +} + +func newViper(flags *pflag.FlagSet) (*viper.Viper, error) { v := viper.New() + v.AutomaticEnv() v.SetEnvPrefix(Prefix) v.AllowEmptyEnv(true) v.SetConfigType("yaml") v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + if err := bindFlags(v, flags); err != nil { + return nil, err + } + + setDefaults(v, flags) + + if v.IsSet(cfgServer+".0."+cfgTLSKeyFile) && v.IsSet(cfgServer+".0."+cfgTLSCertFile) { + v.Set(cfgServer+".0."+cfgTLSEnabled, true) + } + + return v, nil +} + +func settings() *appCfg { // flags setup: flags := pflag.NewFlagSet("commandline", pflag.ExitOnError) flags.SetOutput(os.Stdout) @@ -228,89 +287,17 @@ func settings() *viper.Viper { flags.String(cmdListenAddress, "0.0.0.0:8080", "addresses to listen") flags.String(cfgTLSCertFile, "", "TLS certificate path") flags.String(cfgTLSKeyFile, "", "TLS key path") - peers := flags.StringArrayP(cfgPeers, "p", nil, "FrostFS nodes") + flags.StringArrayP(cfgPeers, "p", nil, "FrostFS nodes") - resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order") - - // set defaults: - - // logger: - v.SetDefault(cfgLoggerLevel, "debug") - v.SetDefault(cfgLoggerDestination, "stdout") - v.SetDefault(cfgLoggerSamplingEnabled, false) - v.SetDefault(cfgLoggerSamplingThereafter, 100) - v.SetDefault(cfgLoggerSamplingInitial, 100) - v.SetDefault(cfgLoggerSamplingInterval, defaultLoggerSamplerInterval) - - // pool: - v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) - - // frostfs: - v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut) - - // web-server: - v.SetDefault(cfgWebReadBufferSize, 4096) - v.SetDefault(cfgWebWriteBufferSize, 4096) - v.SetDefault(cfgWebReadTimeout, time.Minute*10) - v.SetDefault(cfgWebWriteTimeout, time.Minute*5) - v.SetDefault(cfgWebStreamRequestBody, true) - v.SetDefault(cfgWebMaxRequestBodySize, fasthttp.DefaultMaxRequestBodySize) - - v.SetDefault(cfgWorkerPoolSize, 1000) - // upload header - v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) - - // metrics - v.SetDefault(cfgPprofAddress, "localhost:8083") - v.SetDefault(cfgPrometheusAddress, "localhost:8084") - - // resolve bucket - v.SetDefault(cfgResolveNamespaceHeader, defaultNamespaceHeader) - v.SetDefault(cfgResolveDefaultNamespaces, []string{"", "root"}) - - // multinet - v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay) - - // Binding flags - if err := v.BindPFlag(cfgPprofEnabled, flags.Lookup(cmdPprof)); err != nil { - panic(err) - } - if err := v.BindPFlag(cfgPrometheusEnabled, flags.Lookup(cmdMetrics)); err != nil { - panic(err) - } - - if err := v.BindPFlag(cfgWalletPath, flags.Lookup(cmdWallet)); err != nil { - panic(err) - } - - if err := v.BindPFlag(cfgWalletAddress, flags.Lookup(cmdAddress)); err != nil { - panic(err) - } - - if err := v.BindPFlags(flags); err != nil { - panic(err) - } - - if err := v.BindPFlag(cfgServer+".0.address", flags.Lookup(cmdListenAddress)); err != nil { - panic(err) - } - if err := v.BindPFlag(cfgServer+".0."+cfgTLSKeyFile, flags.Lookup(cfgTLSKeyFile)); err != nil { - panic(err) - } - if err := v.BindPFlag(cfgServer+".0."+cfgTLSCertFile, flags.Lookup(cfgTLSCertFile)); err != nil { - panic(err) - } + flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order") if err := flags.Parse(os.Args); err != nil { panic(err) } - if v.IsSet(cfgServer+".0."+cfgTLSKeyFile) && v.IsSet(cfgServer+".0."+cfgTLSCertFile) { - v.Set(cfgServer+".0."+cfgTLSEnabled, true) - } - - if resolveMethods != nil { - v.SetDefault(cfgResolveOrder, *resolveMethods) + v, err := newViper(flags) + if err != nil { + panic(fmt.Errorf("bind flags: %w", err)) } switch { @@ -355,15 +342,97 @@ func settings() *viper.Viper { panic(err) } - if peers != nil && len(*peers) > 0 { - for i := range *peers { - v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".address", (*peers)[i]) + return &appCfg{ + flags: flags, + settings: v, + } +} + +func setDefaults(v *viper.Viper, flags *pflag.FlagSet) { + // set defaults: + + // logger: + v.SetDefault(cfgLoggerLevel, "debug") + v.SetDefault(cfgLoggerDestination, "stdout") + v.SetDefault(cfgLoggerSamplingEnabled, false) + v.SetDefault(cfgLoggerSamplingThereafter, 100) + v.SetDefault(cfgLoggerSamplingInitial, 100) + v.SetDefault(cfgLoggerSamplingInterval, defaultLoggerSamplerInterval) + + // pool: + v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) + + // frostfs: + v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut) + + // web-server: + v.SetDefault(cfgWebReadBufferSize, 4096) + v.SetDefault(cfgWebWriteBufferSize, 4096) + v.SetDefault(cfgWebReadTimeout, time.Minute*10) + v.SetDefault(cfgWebWriteTimeout, time.Minute*5) + v.SetDefault(cfgWebStreamRequestBody, true) + v.SetDefault(cfgWebMaxRequestBodySize, fasthttp.DefaultMaxRequestBodySize) + + v.SetDefault(cfgWorkerPoolSize, 1000) + // upload header + v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) + + // metrics + v.SetDefault(cfgPprofAddress, "localhost:8083") + v.SetDefault(cfgPrometheusAddress, "localhost:8084") + + // resolve bucket + v.SetDefault(cfgResolveNamespaceHeader, defaultNamespaceHeader) + v.SetDefault(cfgResolveDefaultNamespaces, []string{"", "root"}) + + // multinet + v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay) + + if resolveMethods, err := flags.GetStringSlice(cfgResolveOrder); err == nil { + v.SetDefault(cfgResolveOrder, resolveMethods) + } + + if peers, err := flags.GetStringArray(cfgPeers); err == nil { + for i := range peers { + v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".address", peers[i]) v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".weight", 1) v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".priority", 1) } } +} - return v +func bindFlags(v *viper.Viper, flags *pflag.FlagSet) error { + // Binding flags + if err := v.BindPFlag(cfgPprofEnabled, flags.Lookup(cmdPprof)); err != nil { + return err + } + if err := v.BindPFlag(cfgPrometheusEnabled, flags.Lookup(cmdMetrics)); err != nil { + return err + } + + if err := v.BindPFlag(cfgWalletPath, flags.Lookup(cmdWallet)); err != nil { + return err + } + + if err := v.BindPFlag(cfgWalletAddress, flags.Lookup(cmdAddress)); err != nil { + return err + } + + if err := v.BindPFlags(flags); err != nil { + return err + } + + if err := v.BindPFlag(cfgServer+".0.address", flags.Lookup(cmdListenAddress)); err != nil { + return err + } + if err := v.BindPFlag(cfgServer+".0."+cfgTLSKeyFile, flags.Lookup(cfgTLSKeyFile)); err != nil { + return err + } + if err := v.BindPFlag(cfgServer+".0."+cfgTLSCertFile, flags.Lookup(cfgTLSCertFile)); err != nil { + return err + } + + return nil } func readInConfig(v *viper.Viper) error { @@ -616,7 +685,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { } func (a *app) initPools(ctx context.Context) { - key, err := getFrostFSKey(a.cfg, a.log) + key, err := getFrostFSKey(a.config(), a.log) if err != nil { a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) } @@ -628,40 +697,40 @@ func (a *app) initPools(ctx context.Context) { prmTree.SetKey(key) a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) - for _, peer := range fetchPeers(a.log, a.cfg) { + for _, peer := range fetchPeers(a.log, a.config()) { prm.AddNode(peer) prmTree.AddNode(peer) } - connTimeout := a.cfg.GetDuration(cfgConTimeout) + connTimeout := a.config().GetDuration(cfgConTimeout) if connTimeout <= 0 { connTimeout = defaultConnectTimeout } prm.SetNodeDialTimeout(connTimeout) prmTree.SetNodeDialTimeout(connTimeout) - streamTimeout := a.cfg.GetDuration(cfgStreamTimeout) + streamTimeout := a.config().GetDuration(cfgStreamTimeout) if streamTimeout <= 0 { streamTimeout = defaultStreamTimeout } prm.SetNodeStreamTimeout(streamTimeout) prmTree.SetNodeStreamTimeout(streamTimeout) - healthCheckTimeout := a.cfg.GetDuration(cfgReqTimeout) + healthCheckTimeout := a.config().GetDuration(cfgReqTimeout) if healthCheckTimeout <= 0 { healthCheckTimeout = defaultRequestTimeout } prm.SetHealthcheckTimeout(healthCheckTimeout) prmTree.SetHealthcheckTimeout(healthCheckTimeout) - rebalanceInterval := a.cfg.GetDuration(cfgRebalance) + rebalanceInterval := a.config().GetDuration(cfgRebalance) if rebalanceInterval <= 0 { rebalanceInterval = defaultRebalanceTimer } prm.SetClientRebalanceInterval(rebalanceInterval) prmTree.SetClientRebalanceInterval(rebalanceInterval) - errorThreshold := a.cfg.GetUint32(cfgPoolErrorThreshold) + errorThreshold := a.config().GetUint32(cfgPoolErrorThreshold) if errorThreshold <= 0 { errorThreshold = defaultPoolErrorThreshold } @@ -669,7 +738,7 @@ func (a *app) initPools(ctx context.Context) { prm.SetLogger(a.log) prmTree.SetLogger(a.log) - prmTree.SetMaxRequestAttempts(a.cfg.GetInt(cfgTreePoolMaxAttempts)) + prmTree.SetMaxRequestAttempts(a.config().GetInt(cfgTreePoolMaxAttempts)) interceptors := []grpc.DialOption{ grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), @@ -688,8 +757,8 @@ func (a *app) initPools(ctx context.Context) { a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) } - if a.cfg.GetBool(cfgFeaturesTreePoolNetmapSupport) { - prmTree.SetNetMapInfoSource(frostfs.NewSource(frostfs.NewFrostFS(p), cache.NewNetmapCache(getNetmapCacheOptions(a.cfg, a.log)), a.bucketCache, a.log)) + if a.config().GetBool(cfgFeaturesTreePoolNetmapSupport) { + prmTree.SetNetMapInfoSource(frostfs.NewSource(frostfs.NewFrostFS(p), cache.NewNetmapCache(getNetmapCacheOptions(a.config(), a.log)), a.bucketCache, a.log)) } treePool, err := treepool.NewPool(prmTree) diff --git a/cmd/http-gw/settings_test.go b/cmd/http-gw/settings_test.go new file mode 100644 index 0000000..13bd50d --- /dev/null +++ b/cmd/http-gw/settings_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "os" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + "github.com/stretchr/testify/require" +) + +func TestConfigReload(t *testing.T) { + f, err := os.CreateTemp("", "conf") + require.NoError(t, err) + defer func() { + require.NoError(t, os.Remove(f.Name())) + }() + + confData := ` +pprof: + enabled: true + +resolve_bucket: + default_namespaces: [""] + +resolve_order: + - nns +` + + _, err = f.WriteString(confData) + require.NoError(t, err) + require.NoError(t, f.Close()) + + cfg := settings() + + require.NoError(t, cfg.flags.Parse([]string{"--config", f.Name(), "--connect_timeout", "15s"})) + require.NoError(t, cfg.reload()) + + require.True(t, cfg.config().GetBool(cfgPprofEnabled)) + require.Equal(t, []string{""}, cfg.config().GetStringSlice(cfgResolveDefaultNamespaces)) + require.Equal(t, []string{resolver.NNSResolver}, cfg.config().GetStringSlice(cfgResolveOrder)) + require.Equal(t, 15*time.Second, cfg.config().GetDuration(cfgConTimeout)) + + require.NoError(t, os.Truncate(f.Name(), 0)) + require.NoError(t, cfg.reload()) + + require.False(t, cfg.config().GetBool(cfgPprofEnabled)) + require.Equal(t, []string{"", "root"}, cfg.config().GetStringSlice(cfgResolveDefaultNamespaces)) + require.Equal(t, []string{resolver.NNSResolver, resolver.DNSResolver}, cfg.config().GetStringSlice(cfgResolveOrder)) + require.Equal(t, 15*time.Second, cfg.config().GetDuration(cfgConTimeout)) +} + +func TestSetTLSEnabled(t *testing.T) { + cfg := settings() + + require.NoError(t, cfg.flags.Parse([]string{"--" + cfgTLSCertFile, "tls.crt", "--" + cfgTLSKeyFile, "tls.key"})) + require.NoError(t, cfg.reload()) + + require.True(t, cfg.config().GetBool(cfgServer+".0."+cfgTLSEnabled)) +} From 8362cd696e95d1e3b3d2bcce83deec08b0895918 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Tue, 28 Jan 2025 13:42:48 +0300 Subject: [PATCH 513/548] [#199] Port release v0.32.1 changelog Signed-off-by: Marina Biryukova --- CHANGELOG.md | 8 +++++++- VERSION | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd37815..ee0659c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ This document outlines major changes between releases. ### Added - Add handling quota limit reached error (#187) +## [0.32.1] - 2025-01-27 + +### Fixed +- SIGHUP panic (#198) + ## [0.32.0] - Khumbu - 2024-12-20 ### Fixed @@ -187,4 +192,5 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.2...v0.30.3 [0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.3...v0.31.0 [0.32.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...v0.32.0 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.0...master \ No newline at end of file +[0.32.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.0...v0.32.1 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.1...master \ No newline at end of file diff --git a/VERSION b/VERSION index 420000f..0e22fde 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.32.0 +v0.32.1 From 72e5d645b9abb5647910ad2aa7ca48b01fe26037 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 27 Jan 2025 14:23:31 +0300 Subject: [PATCH 514/548] [#194] Fix updateServers finding logic Signed-off-by: Denis Kirillov --- cmd/http-gw/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index ac20d29..1bacbed 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -894,8 +894,8 @@ func (a *app) updateServers() error { if err := ser.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { return fmt.Errorf("failed to update tls certs: %w", err) } - found = true } + found = true } else if unbind := a.updateUnbindServerInfo(serverInfo); unbind { found = true } From 7e48ca626e129ec1526602fd6fc786d41dfeb44d Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 30 Jan 2025 16:38:02 +0300 Subject: [PATCH 515/548] [#202] Bump SDK version to the latest master Contains fixes: - memory leak in gRPC client, - panic and deadlock in tree pool. Signed-off-by: Alex Vanin --- go.mod | 22 ++++++++++------------ go.sum | 44 ++++++++++++++++++++++++-------------------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 46fe3bc..d0bfcbb 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 @@ -22,13 +22,13 @@ require ( github.com/testcontainers/testcontainers-go v0.35.0 github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 github.com/valyala/fasthttp v1.34.0 - go.opentelemetry.io/otel v1.28.0 - go.opentelemetry.io/otel/trace v1.28.0 + go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/net v0.26.0 + golang.org/x/net v0.30.0 golang.org/x/sys v0.28.0 - google.golang.org/grpc v1.66.2 + google.golang.org/grpc v1.69.2 ) require ( @@ -68,12 +68,10 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/ipfs/go-cid v0.0.7 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -122,8 +120,8 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.31.0 // indirect @@ -131,9 +129,9 @@ require ( golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/protobuf v1.36.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect diff --git a/go.sum b/go.sum index e5bfc09..8eb4ea9 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae h1:7gvuOTmS3oaOM79JkHWWlsvGqIRqsum5KnOI1TYqfn0= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae/go.mod h1:dbWUc5jOBTXVvssCLCYxkkSTL9jgLr1KruGP2FMAfiM= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= @@ -172,6 +172,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -418,8 +420,8 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= @@ -428,12 +430,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMey go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -528,8 +532,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -745,10 +749,10 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -765,8 +769,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= -google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= +google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -777,8 +781,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 1779593f46580a3d4ad11009f3a15e7472e99719 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 30 Jan 2025 16:39:27 +0300 Subject: [PATCH 516/548] [#203] Port changelog from support branch Signed-off-by: Alex Vanin --- CHANGELOG.md | 8 +++++++- VERSION | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee0659c..30fcf7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ This document outlines major changes between releases. ### Added - Add handling quota limit reached error (#187) +## [0.32.2] - 2025-02-03 + +### Fixed +- Possible memory leak in gRPC client (#202) + ## [0.32.1] - 2025-01-27 ### Fixed @@ -193,4 +198,5 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.3...v0.31.0 [0.32.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...v0.32.0 [0.32.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.0...v0.32.1 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.1...master \ No newline at end of file +[0.32.2]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.1...v0.32.2 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.2...master \ No newline at end of file diff --git a/VERSION b/VERSION index 0e22fde..c6a2605 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.32.1 +v0.32.2 From 76bd6ea40f9ac78fb5e83f0fd20e116866aba7ec Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 7 Feb 2025 12:58:01 +0300 Subject: [PATCH 517/548] [#206] Bump go version in vulncheck go1.22.11 triggers GO-2025-3447 but this is applicable only for ppc64le platform. Signed-off-by: Alex Vanin --- .forgejo/workflows/vulncheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 529e8a8..5cb6e73 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -16,7 +16,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22.11' + go-version: '1.22.12' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest From 6a4d3206bd18ba9b1881c1f0f4be3aafd9979554 Mon Sep 17 00:00:00 2001 From: Aleksey Kravchenko Date: Tue, 28 Jan 2025 14:42:40 +0300 Subject: [PATCH 518/548] [#195] Add tags support Signed-off-by: Aleksey Kravchenko --- cmd/http-gw/app.go | 203 ++++++++++++++++++++--------- cmd/http-gw/logger.go | 174 +++++++++++++++++++++++++ cmd/http-gw/settings.go | 165 +++++++---------------- config/config.env | 2 + config/config.yaml | 4 + docs/gate-configuration.md | 29 +++++ internal/cache/buckets.go | 4 +- internal/cache/netmap.go | 2 +- internal/handler/browse.go | 6 +- internal/handler/download.go | 27 ++-- internal/handler/filter.go | 3 +- internal/handler/handler.go | 26 ++-- internal/handler/head.go | 6 +- internal/handler/multipart.go | 6 +- internal/handler/multipart_test.go | 5 +- internal/handler/reader.go | 5 +- internal/handler/upload.go | 27 ++-- internal/handler/utils.go | 4 +- internal/logs/logs.go | 149 ++++++++++++--------- internal/net/event_handler.go | 6 +- internal/service/frostfs/source.go | 2 +- metrics/service.go | 12 +- 22 files changed, 572 insertions(+), 295 deletions(-) create mode 100644 cmd/http-gw/logger.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 1bacbed..103c72b 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -44,6 +44,7 @@ import ( "github.com/valyala/fasthttp" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "golang.org/x/exp/slices" ) @@ -51,7 +52,6 @@ type ( app struct { ctx context.Context log *zap.Logger - logLevel zap.AtomicLevel pool *pool.Pool treePool *treepool.Pool key *keys.PrivateKey @@ -94,6 +94,7 @@ type ( reconnectInterval time.Duration dialerSource *internalnet.DialerSource workerPoolSize int + logLevelConfig *logLevelConfig mu sync.RWMutex defaultTimestamp bool @@ -113,6 +114,15 @@ type ( enableFilepathFallback bool } + tagsConfig struct { + tagLogs sync.Map + } + + logLevelConfig struct { + logLevel zap.AtomicLevel + tagsConfig *tagsConfig + } + CORS struct { AllowOrigin string AllowMethods []string @@ -123,14 +133,91 @@ type ( } ) +func newLogLevel(v *viper.Viper) zap.AtomicLevel { + ll, err := getLogLevel(v) + if err != nil { + panic(err.Error()) + } + atomicLogLevel := zap.NewAtomicLevel() + atomicLogLevel.SetLevel(ll) + return atomicLogLevel +} + +func newTagsConfig(v *viper.Viper, ll zapcore.Level) *tagsConfig { + var t tagsConfig + if err := t.update(v, ll); err != nil { + // panic here is analogue of the similar panic during common log level initialization. + panic(err.Error()) + } + return &t +} + +func newLogLevelConfig(lvl zap.AtomicLevel, tagsConfig *tagsConfig) *logLevelConfig { + return &logLevelConfig{ + logLevel: lvl, + tagsConfig: tagsConfig, + } +} + +func (l *logLevelConfig) update(cfg *viper.Viper, log *zap.Logger) { + if lvl, err := getLogLevel(cfg); err != nil { + log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) + } else { + l.logLevel.SetLevel(lvl) + } + + if err := l.tagsConfig.update(cfg, l.logLevel.Level()); err != nil { + log.Warn(logs.TagsLogConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) + } +} + +func (t *tagsConfig) LevelEnabled(tag string, tgtLevel zapcore.Level) bool { + lvl, ok := t.tagLogs.Load(tag) + if !ok { + return false + } + + return lvl.(zapcore.Level).Enabled(tgtLevel) +} + +func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error { + tags, err := fetchLogTagsConfig(cfg, ll) + if err != nil { + return err + } + + t.tagLogs.Range(func(key, value any) bool { + k := key.(string) + v := value.(zapcore.Level) + + if lvl, ok := tags[k]; ok { + if lvl != v { + t.tagLogs.Store(key, lvl) + } + } else { + t.tagLogs.Delete(key) + delete(tags, k) + } + return true + }) + + for k, v := range tags { + t.tagLogs.Store(k, v) + } + + return nil +} + func newApp(ctx context.Context, cfg *appCfg) App { logSettings := &loggerSettings{} - log := pickLogger(cfg.config(), logSettings) + logLevel := newLogLevel(cfg.config()) + tagConfig := newTagsConfig(cfg.config(), logLevel.Level()) + logConfig := newLogLevelConfig(logLevel, tagConfig) + log := pickLogger(cfg.config(), logConfig.logLevel, logSettings, tagConfig) a := &app{ ctx: ctx, log: log.logger, - logLevel: log.lvl, cfg: cfg, loggerSettings: logSettings, webServer: new(fasthttp.Server), @@ -138,7 +225,7 @@ func newApp(ctx context.Context, cfg *appCfg) App { bucketCache: cache.NewBucketCache(getBucketCacheOptions(cfg.config(), log.logger), cfg.config().GetBool(cfgFeaturesTreePoolNetmapSupport)), } - a.initAppSettings() + a.initAppSettings(logConfig) // -- setup FastHTTP server -- a.webServer.Name = "frost-http-gw" @@ -172,11 +259,12 @@ func (a *app) config() *viper.Viper { return a.cfg.config() } -func (a *app) initAppSettings() { +func (a *app) initAppSettings(lc *logLevelConfig) { a.settings = &appSettings{ reconnectInterval: fetchReconnectInterval(a.config()), dialerSource: getDialerSource(a.log, a.config()), workerPoolSize: a.config().GetInt(cfgWorkerPoolSize), + logLevelConfig: lc, } a.settings.update(a.config(), a.log) } @@ -324,7 +412,7 @@ func (a *app) initResolver() { var err error a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) if err != nil { - a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err), logs.TagField(logs.TagApp)) } } @@ -338,11 +426,12 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) { order := a.config().GetStringSlice(cfgResolveOrder) if resolveCfg.RPCAddress == "" { order = remove(order, resolver.NNSResolver) - a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided) + a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided, logs.TagField(logs.TagApp)) } if len(order) == 0 { - a.log.Info(logs.ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty) + a.log.Info(logs.ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty, + logs.TagField(logs.TagApp)) } return order, resolveCfg @@ -357,7 +446,7 @@ func (a *app) initMetrics() { func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics { if !enabled { - logger.Warn(logs.MetricsAreDisabled) + logger.Warn(logs.MetricsAreDisabled, logs.TagField(logs.TagApp)) } return &gateMetrics{ logger: logger, @@ -375,7 +464,7 @@ func (m *gateMetrics) isEnabled() bool { func (m *gateMetrics) SetEnabled(enabled bool) { if !enabled { - m.logger.Warn(logs.MetricsAreDisabled) + m.logger.Warn(logs.MetricsAreDisabled, logs.TagField(logs.TagApp)) } m.mu.Lock() @@ -438,7 +527,7 @@ func getFrostFSKey(cfg *viper.Viper, log *zap.Logger) (*keys.PrivateKey, error) walletPath := cfg.GetString(cfgWalletPath) if len(walletPath) == 0 { - log.Info(logs.NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun) + log.Info(logs.NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun, logs.TagField(logs.TagApp)) key, err := keys.NewPrivateKey() if err != nil { return nil, err @@ -495,7 +584,10 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*keys } func (a *app) Wait() { - a.log.Info(logs.StartingApplication, zap.String("app_name", "frostfs-http-gw"), zap.String("version", Version)) + a.log.Info(logs.StartingApplication, + zap.String("app_name", "frostfs-http-gw"), + zap.String("version", Version), + logs.TagField(logs.TagApp)) a.metrics.SetVersion(Version) a.setHealthStatus() @@ -526,10 +618,10 @@ func (a *app) Serve() { for i := range servs { go func(i int) { - a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address())) + a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address()), logs.TagField(logs.TagApp)) if err := a.webServer.Serve(servs[i].Listener()); err != nil && err != http.ErrServerClosed { a.metrics.MarkUnhealthy(servs[i].Address()) - a.log.Fatal(logs.ListenAndServe, zap.Error(err)) + a.log.Fatal(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp)) } }(i) } @@ -551,7 +643,7 @@ LOOP: } } - a.log.Info(logs.ShuttingDownWebServer, zap.Error(a.webServer.Shutdown())) + a.log.Info(logs.ShuttingDownWebServer, zap.Error(a.webServer.Shutdown()), logs.TagField(logs.TagApp)) a.metrics.Shutdown() a.stopServices() @@ -561,7 +653,7 @@ LOOP: func (a *app) initWorkerPool() *ants.Pool { workerPool, err := ants.NewPool(a.settings.workerPoolSize) if err != nil { - a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err), logs.TagField(logs.TagApp)) } return workerPool } @@ -572,37 +664,33 @@ func (a *app) shutdownTracing() { defer cancel() if err := tracing.Shutdown(shdnCtx); err != nil { - a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err)) + a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err), logs.TagField(logs.TagApp)) } } func (a *app) configReload(ctx context.Context) { - a.log.Info(logs.SIGHUPConfigReloadStarted) + a.log.Info(logs.SIGHUPConfigReloadStarted, logs.TagField(logs.TagApp)) if !a.config().IsSet(cmdConfig) && !a.config().IsSet(cmdConfigDir) { - a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed) + a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed, logs.TagField(logs.TagApp)) return } if err := a.cfg.reload(); err != nil { - a.log.Warn(logs.FailedToReloadConfig, zap.Error(err)) + a.log.Warn(logs.FailedToReloadConfig, zap.Error(err), logs.TagField(logs.TagApp)) return } - if lvl, err := getLogLevel(a.config()); err != nil { - a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err)) - } else { - a.logLevel.SetLevel(lvl) - } + a.settings.logLevelConfig.update(a.cfg.settings, a.log) if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil { - a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err)) + a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) } if err := a.resolver.UpdateResolvers(a.getResolverConfig()); err != nil { - a.log.Warn(logs.FailedToUpdateResolvers, zap.Error(err)) + a.log.Warn(logs.FailedToUpdateResolvers, zap.Error(err), logs.TagField(logs.TagApp)) } if err := a.updateServers(); err != nil { - a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err)) + a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err), logs.TagField(logs.TagApp)) } a.setRuntimeParameters() @@ -616,7 +704,7 @@ func (a *app) configReload(ctx context.Context) { a.initTracing(ctx) a.setHealthStatus() - a.log.Info(logs.SIGHUPConfigReloadCompleted) + a.log.Info(logs.SIGHUPConfigReloadCompleted, logs.TagField(logs.TagApp)) } func (a *app) startServices() { @@ -654,20 +742,20 @@ func (a *app) configureRouter(h *handler.Handler) { r.POST("/upload/{cid}", a.addMiddlewares(h.Upload)) r.OPTIONS("/upload/{cid}", a.addPreflight()) - a.log.Info(logs.AddedPathUploadCid) + a.log.Info(logs.AddedPathUploadCid, logs.TagField(logs.TagApp)) r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(h.DownloadByAddressOrBucketName)) r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(h.HeadByAddressOrBucketName)) r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight()) - a.log.Info(logs.AddedPathGetCidOid) + a.log.Info(logs.AddedPathGetCidOid, logs.TagField(logs.TagApp)) r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.DownloadByAttribute)) r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.HeadByAttribute)) r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) - a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) + a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal, logs.TagField(logs.TagApp)) r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZip)) r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadTar)) r.OPTIONS("/tar/{cid}/{prefix:*}", a.addPreflight()) - a.log.Info(logs.AddedPathZipCidPrefix) + a.log.Info(logs.AddedPathZipCidPrefix, logs.TagField(logs.TagApp)) a.webServer.Handler = r.Handler } @@ -756,14 +844,11 @@ func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { reqCtx = utils.SetReqLog(reqCtx, log) utils.SetContextToRequest(reqCtx, req) - fields := []zap.Field{ - zap.String("remote", req.RemoteAddr().String()), + log.Info(logs.Request, zap.String("remote", req.RemoteAddr().String()), zap.ByteString("method", req.Method()), zap.ByteString("path", req.Path()), zap.ByteString("query", req.QueryArgs().QueryString()), - } - - log.Info(logs.Request, fields...) + logs.TagField(logs.TagDatapath)) h(req) } } @@ -807,7 +892,7 @@ func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { if err != nil { log := utils.GetReqLogOrDefault(reqCtx, a.log) - log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err)) + log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err), logs.TagField(logs.TagDatapath)) handler.ResponseError(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -866,17 +951,17 @@ func (a *app) initServers(ctx context.Context) { if err != nil { a.unbindServers = append(a.unbindServers, serverInfo) a.metrics.MarkUnhealthy(serverInfo.Address) - a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...) + a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err), logs.TagField(logs.TagApp))...) continue } a.metrics.MarkHealthy(serverInfo.Address) a.servers = append(a.servers, srv) - a.log.Info(logs.AddServer, fields...) + a.log.Info(logs.AddServer, append(fields, logs.TagField(logs.TagApp))...) } if len(a.servers) == 0 { - a.log.Fatal(logs.NoHealthyServers) + a.log.Fatal(logs.NoHealthyServers, logs.TagField(logs.TagApp)) } } @@ -950,13 +1035,14 @@ func (a *app) initTracing(ctx context.Context) { if trustedCa := a.config().GetString(cfgTracingTrustedCa); trustedCa != "" { caBytes, err := os.ReadFile(trustedCa) if err != nil { - a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp)) return } certPool := x509.NewCertPool() ok := certPool.AppendCertsFromPEM(caBytes) if !ok { - a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert")) + a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert"), + logs.TagField(logs.TagApp)) return } cfg.ServerCaCertPool = certPool @@ -964,24 +1050,24 @@ func (a *app) initTracing(ctx context.Context) { attributes, err := fetchTracingAttributes(a.config()) if err != nil { - a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp)) return } cfg.Attributes = attributes updated, err := tracing.Setup(ctx, cfg) if err != nil { - a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp)) } if updated { - a.log.Info(logs.TracingConfigUpdated) + a.log.Info(logs.TracingConfigUpdated, logs.TagField(logs.TagApp)) } } func (a *app) setRuntimeParameters() { if len(os.Getenv("GOMEMLIMIT")) != 0 { // default limit < yaml limit < app env limit < GOMEMLIMIT - a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) + a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT, logs.TagField(logs.TagApp)) return } @@ -990,7 +1076,8 @@ func (a *app) setRuntimeParameters() { if softMemoryLimit != previous { a.log.Info(logs.RuntimeSoftMemoryLimitUpdated, zap.Int64("new_value", softMemoryLimit), - zap.Int64("old_value", previous)) + zap.Int64("old_value", previous), + logs.TagField(logs.TagApp)) } } @@ -1016,34 +1103,32 @@ func (a *app) tryReconnect(ctx context.Context, sr *fasthttp.Server) bool { a.mu.Lock() defer a.mu.Unlock() - a.log.Info(logs.ServerReconnecting) + a.log.Info(logs.ServerReconnecting, logs.TagField(logs.TagApp)) var failedServers []ServerInfo for _, serverInfo := range a.unbindServers { - fields := []zap.Field{ - zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled), - zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile), - } - srv, err := newServer(ctx, serverInfo) if err != nil { - a.log.Warn(logs.ServerReconnectFailed, zap.Error(err)) + a.log.Warn(logs.ServerReconnectFailed, zap.Error(err), logs.TagField(logs.TagApp)) failedServers = append(failedServers, serverInfo) a.metrics.MarkUnhealthy(serverInfo.Address) continue } go func() { - a.log.Info(logs.StartingServer, zap.String("address", srv.Address())) + a.log.Info(logs.StartingServer, zap.String("address", srv.Address()), logs.TagField(logs.TagApp)) a.metrics.MarkHealthy(serverInfo.Address) if err = sr.Serve(srv.Listener()); err != nil && !errors.Is(err, http.ErrServerClosed) { - a.log.Warn(logs.ListenAndServe, zap.Error(err)) + a.log.Warn(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp)) a.metrics.MarkUnhealthy(serverInfo.Address) } }() a.servers = append(a.servers, srv) - a.log.Info(logs.ServerReconnectedSuccessfully, fields...) + a.log.Info(logs.ServerReconnectedSuccessfully, + zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled), + zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile), + logs.TagField(logs.TagApp)) } a.unbindServers = failedServers diff --git a/cmd/http-gw/logger.go b/cmd/http-gw/logger.go new file mode 100644 index 0000000..91105f7 --- /dev/null +++ b/cmd/http-gw/logger.go @@ -0,0 +1,174 @@ +package main + +import ( + "fmt" + "os" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/zapjournald" + "github.com/spf13/viper" + "github.com/ssgreg/journald" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func getLogLevel(v *viper.Viper) (zapcore.Level, error) { + var lvl zapcore.Level + lvlStr := v.GetString(cfgLoggerLevel) + err := lvl.UnmarshalText([]byte(lvlStr)) + if err != nil { + return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+ + "value should be one of %v", lvlStr, err, [...]zapcore.Level{ + zapcore.DebugLevel, + zapcore.InfoLevel, + zapcore.WarnLevel, + zapcore.ErrorLevel, + zapcore.DPanicLevel, + zapcore.PanicLevel, + zapcore.FatalLevel, + }) + } + return lvl, nil +} + +var _ zapcore.Core = (*zapCoreTagFilterWrapper)(nil) + +type zapCoreTagFilterWrapper struct { + core zapcore.Core + settings TagFilterSettings + extra []zap.Field +} + +type TagFilterSettings interface { + LevelEnabled(tag string, lvl zapcore.Level) bool +} + +func (c *zapCoreTagFilterWrapper) Enabled(level zapcore.Level) bool { + return c.core.Enabled(level) +} + +func (c *zapCoreTagFilterWrapper) With(fields []zapcore.Field) zapcore.Core { + return &zapCoreTagFilterWrapper{ + core: c.core.With(fields), + settings: c.settings, + extra: append(c.extra, fields...), + } +} + +func (c *zapCoreTagFilterWrapper) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if c.core.Enabled(entry.Level) { + return checked.AddCore(entry, c) + } + return checked +} + +func (c *zapCoreTagFilterWrapper) Write(entry zapcore.Entry, fields []zapcore.Field) error { + if c.shouldSkip(entry, fields) || c.shouldSkip(entry, c.extra) { + return nil + } + + return c.core.Write(entry, fields) +} + +func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field) bool { + for _, field := range fields { + if field.Key == logs.TagFieldName && field.Type == zapcore.StringType { + if !c.settings.LevelEnabled(field.String, entry.Level) { + return true + } + break + } + } + + return false +} + +func (c *zapCoreTagFilterWrapper) Sync() error { + return c.core.Sync() +} + +func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) zapcore.Core { + core = &zapCoreTagFilterWrapper{ + core: core, + settings: tagSetting, + } + + if v.GetBool(cfgLoggerSamplingEnabled) { + core = zapcore.NewSamplerWithOptions(core, + v.GetDuration(cfgLoggerSamplingInterval), + v.GetInt(cfgLoggerSamplingInitial), + v.GetInt(cfgLoggerSamplingThereafter), + zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) { + if dec&zapcore.LogDropped > 0 { + loggerSettings.DroppedLogsInc() + } + })) + } + + return core +} + +func newLogEncoder() zapcore.Encoder { + c := zap.NewProductionEncoderConfig() + c.EncodeTime = zapcore.ISO8601TimeEncoder + + return zapcore.NewConsoleEncoder(c) +} + +// newStdoutLogger constructs a zap.Logger instance for current application. +// Panics on failure. +// +// Logger is built from zap's production logging configuration with: +// - parameterized level (debug by default) +// - console encoding +// - ISO8601 time encoding +// +// Logger records a stack trace for all messages at or above fatal level. +// +// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. +func newStdoutLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) *Logger { + stdout := zapcore.AddSync(os.Stderr) + + consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, lvl) + consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, loggerSettings, tagSetting) + + return &Logger{ + logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), + lvl: lvl, + } +} + +func newJournaldLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) *Logger { + encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields) + + core := zapjournald.NewCore(lvl, encoder, &journald.Journal{}, zapjournald.SyslogFields) + coreWithContext := core.With([]zapcore.Field{ + zapjournald.SyslogFacility(zapjournald.LogDaemon), + zapjournald.SyslogIdentifier(), + zapjournald.SyslogPid(), + }) + + coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, loggerSettings, tagSetting) + + return &Logger{ + logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), + lvl: lvl, + } +} + +type LoggerAppSettings interface { + DroppedLogsInc() +} + +func pickLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger { + dest := v.GetString(cfgLoggerDestination) + + switch dest { + case destinationStdout: + return newStdoutLogger(v, lvl, loggerSettings, tagSettings) + case destinationJournald: + return newJournaldLogger(v, lvl, loggerSettings, tagSettings) + default: + panic(fmt.Sprintf("wrong destination for logger: %s", dest)) + } +} diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 5670f87..d8331ad 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -23,10 +23,8 @@ import ( grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" - "git.frostfs.info/TrueCloudLab/zapjournald" "github.com/spf13/pflag" "github.com/spf13/viper" - "github.com/ssgreg/journald" "github.com/valyala/fasthttp" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -111,6 +109,11 @@ const ( cfgLoggerSamplingThereafter = "logger.sampling.thereafter" cfgLoggerSamplingInterval = "logger.sampling.interval" + cfgLoggerTags = "logger.tags" + cfgLoggerTagsPrefixTmpl = cfgLoggerTags + ".%d." + cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "name" + cfgLoggerTagsLevelTmpl = cfgLoggerTagsPrefixTmpl + "level" + // Wallet. cfgWalletPassphrase = "wallet.passphrase" cfgWalletPath = "wallet.path" @@ -193,6 +196,8 @@ var ignore = map[string]struct{}{ cmdVersion: {}, } +var defaultTags = []string{logs.TagApp, logs.TagDatapath} + type Logger struct { logger *zap.Logger lvl zap.AtomicLevel @@ -499,112 +504,33 @@ func mergeConfig(v *viper.Viper, fileName string) error { return v.MergeConfig(cfgFile) } -type LoggerAppSettings interface { - DroppedLogsInc() -} +func fetchLogTagsConfig(v *viper.Viper, defaultLvl zapcore.Level) (map[string]zapcore.Level, error) { + res := make(map[string]zapcore.Level) -func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger { - lvl, err := getLogLevel(v) - if err != nil { - panic(err) + for i := 0; ; i++ { + name := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i)) + if name == "" { + break + } + + lvl := defaultLvl + level := v.GetString(fmt.Sprintf(cfgLoggerTagsLevelTmpl, i)) + if level != "" { + if err := lvl.Set(level); err != nil { + return nil, fmt.Errorf("failed to parse log tags config, unknown level: '%s'", level) + } + } + + res[name] = lvl } - dest := v.GetString(cfgLoggerDestination) - - switch dest { - case destinationStdout: - return newStdoutLogger(v, lvl, settings) - case destinationJournald: - return newJournaldLogger(v, lvl, settings) - default: - panic(fmt.Sprintf("wrong destination for logger: %s", dest)) - } -} - -// newStdoutLogger constructs a zap.Logger instance for current application. -// Panics on failure. -// -// Logger is built from zap's production logging configuration with: -// - parameterized level (debug by default) -// - console encoding -// - ISO8601 time encoding -// -// Logger records a stack trace for all messages at or above fatal level. -// -// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. -func newStdoutLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger { - stdout := zapcore.AddSync(os.Stderr) - level := zap.NewAtomicLevelAt(lvl) - - consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level) - consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, settings) - - return &Logger{ - logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), - lvl: level, - } -} - -func newJournaldLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger { - level := zap.NewAtomicLevelAt(lvl) - - encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields) - - core := zapjournald.NewCore(level, encoder, &journald.Journal{}, zapjournald.SyslogFields) - coreWithContext := core.With([]zapcore.Field{ - zapjournald.SyslogFacility(zapjournald.LogDaemon), - zapjournald.SyslogIdentifier(), - zapjournald.SyslogPid(), - }) - - coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, settings) - - return &Logger{ - logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), - lvl: level, - } -} - -func newLogEncoder() zapcore.Encoder { - c := zap.NewProductionEncoderConfig() - c.EncodeTime = zapcore.ISO8601TimeEncoder - - return zapcore.NewConsoleEncoder(c) -} - -func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, settings LoggerAppSettings) zapcore.Core { - if v.GetBool(cfgLoggerSamplingEnabled) { - core = zapcore.NewSamplerWithOptions(core, - v.GetDuration(cfgLoggerSamplingInterval), - v.GetInt(cfgLoggerSamplingInitial), - v.GetInt(cfgLoggerSamplingThereafter), - zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) { - if dec&zapcore.LogDropped > 0 { - settings.DroppedLogsInc() - } - })) + if len(res) == 0 && !v.IsSet(cfgLoggerTags) { + for _, tag := range defaultTags { + res[tag] = defaultLvl + } } - return core -} - -func getLogLevel(v *viper.Viper) (zapcore.Level, error) { - var lvl zapcore.Level - lvlStr := v.GetString(cfgLoggerLevel) - err := lvl.UnmarshalText([]byte(lvlStr)) - if err != nil { - return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+ - "value should be one of %v", lvlStr, err, [...]zapcore.Level{ - zapcore.DebugLevel, - zapcore.InfoLevel, - zapcore.WarnLevel, - zapcore.ErrorLevel, - zapcore.DPanicLevel, - zapcore.PanicLevel, - zapcore.FatalLevel, - }) - } - return lvl, nil + return res, nil } func fetchReconnectInterval(cfg *viper.Viper) time.Duration { @@ -620,20 +546,19 @@ func fetchIndexPageTemplate(v *viper.Viper, l *zap.Logger) (string, bool) { if !v.GetBool(cfgIndexPageEnabled) { return "", false } - reader, err := os.Open(v.GetString(cfgIndexPageTemplatePath)) if err != nil { - l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err), logs.TagField(logs.TagApp)) return "", true } tmpl, err := io.ReadAll(reader) if err != nil { - l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err), logs.TagField(logs.TagApp)) return "", true } - l.Info(logs.SetCustomIndexPageTemplate) + l.Info(logs.SetCustomIndexPageTemplate, logs.TagField(logs.TagApp)) return string(tmpl), true } @@ -674,7 +599,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { } if _, ok := seen[serverInfo.Address]; ok { - log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address)) + log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address), logs.TagField(logs.TagApp)) continue } seen[serverInfo.Address] = struct{}{} @@ -687,7 +612,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { func (a *app) initPools(ctx context.Context) { key, err := getFrostFSKey(a.config(), a.log) if err != nil { - a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) + a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err), logs.TagField(logs.TagApp)) } var prm pool.InitParameters @@ -695,7 +620,8 @@ func (a *app) initPools(ctx context.Context) { prm.SetKey(&key.PrivateKey) prmTree.SetKey(key) - a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) + a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())), + logs.TagField(logs.TagApp)) for _, peer := range fetchPeers(a.log, a.config()) { prm.AddNode(peer) @@ -750,11 +676,11 @@ func (a *app) initPools(ctx context.Context) { p, err := pool.NewPool(prm) if err != nil { - a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err), logs.TagField(logs.TagApp)) } if err = p.Dial(ctx); err != nil { - a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err), logs.TagField(logs.TagApp)) } if a.config().GetBool(cfgFeaturesTreePoolNetmapSupport) { @@ -763,10 +689,10 @@ func (a *app) initPools(ctx context.Context) { treePool, err := treepool.NewPool(prmTree) if err != nil { - a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err), logs.TagField(logs.TagApp)) } if err = treePool.Dial(ctx); err != nil { - a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err), logs.TagField(logs.TagApp)) } a.pool = p @@ -797,7 +723,8 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam { l.Info(logs.AddedStoragePeer, zap.Int("priority", priority), zap.String("address", address), - zap.Float64("weight", weight)) + zap.Float64("weight", weight), + logs.TagField(logs.TagApp)) } return nodes @@ -836,7 +763,8 @@ func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultV l.Error(logs.InvalidLifetimeUsingDefaultValue, zap.String("parameter", cfgEntry), zap.Duration("value in config", lifetime), - zap.Duration("default", defaultValue)) + zap.Duration("default", defaultValue), + logs.TagField(logs.TagApp)) } else { return lifetime } @@ -852,7 +780,8 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue l.Error(logs.InvalidCacheSizeUsingDefaultValue, zap.String("parameter", cfgEntry), zap.Int("value in config", size), - zap.Int("default", defaultValue)) + zap.Int("default", defaultValue), + logs.TagField(logs.TagApp)) } else { return size } @@ -864,7 +793,7 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSource { source, err := internalnet.NewDialerSource(fetchMultinetConfig(cfg, logger)) if err != nil { - logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err)) + logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err), logs.TagField(logs.TagApp)) } return source } diff --git a/config/config.env b/config/config.env index db619b5..af0eba1 100644 --- a/config/config.env +++ b/config/config.env @@ -20,6 +20,8 @@ HTTP_GW_LOGGER_SAMPLING_ENABLED=false HTTP_GW_LOGGER_SAMPLING_INITIAL=100 HTTP_GW_LOGGER_SAMPLING_THEREAFTER=100 HTTP_GW_LOGGER_SAMPLING_INTERVAL=1s +HTTP_GW_LOGGER_TAGS_0_NAME=app +HTTP_GW_LOGGER_TAGS_1_NAME=datapath HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:443 HTTP_GW_SERVER_0_TLS_ENABLED=false diff --git a/config/config.yaml b/config/config.yaml index a70ec9a..8c51591 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -29,6 +29,10 @@ logger: initial: 100 thereafter: 100 interval: 1s + tags: + - name: app + - name: datapath + level: debug server: - address: 0.0.0.0:8080 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 6aadd1f..4f9bc3b 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -174,6 +174,11 @@ logger: initial: 100 thereafter: 100 interval: 1s + tags: + - name: "app" + level: info + - name: "datapath" + - name: "external_storage_tree" ``` | Parameter | Type | SIGHUP reload | Default value | Description | @@ -184,6 +189,30 @@ logger: | `sampling.initial` | `int` | no | '100' | Sampling count of first log entries. | | `sampling.thereafter` | `int` | no | '100' | Sampling count of entries after an `interval`. | | `sampling.interval` | `duration` | no | '1s' | Sampling interval of messaging similar entries. | +| `sampling.tags` | `[]Tag` | yes | | Tagged log entries that should be additionally logged (available tags see in the next section). | + +## Tags + +There are additional log entries that can hurt performance and can be additionally logged by using `logger.tags` +parameter. Available tags: + +```yaml +tags: + - name: "app" + level: info +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------------------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------| +| `name` | `string` | yes | | Tag name. Possible values see below in `Tag values` section. | +| `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. | + +### Tag values + +* `app` - common application logs (enabled by default). +* `datapath` - main logic of application (enabled by default). +* `external_storage` - external interaction with storage node. +* `external_storage_tree` - external interaction with tree service in storage node. # `web` section diff --git a/internal/cache/buckets.go b/internal/cache/buckets.go index 2fa8f25..91ae5b2 100644 --- a/internal/cache/buckets.go +++ b/internal/cache/buckets.go @@ -72,7 +72,7 @@ func (o *BucketCache) GetByCID(cnrID cid.ID) *data.BucketInfo { key, ok := entry.(string) if !ok { o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), - zap.String("expected", fmt.Sprintf("%T", key))) + zap.String("expected", fmt.Sprintf("%T", key)), logs.TagField(logs.TagDatapath)) return nil } @@ -88,7 +88,7 @@ func (o *BucketCache) get(key string) *data.BucketInfo { result, ok := entry.(*data.BucketInfo) if !ok { o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), - zap.String("expected", fmt.Sprintf("%T", result))) + zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath)) return nil } diff --git a/internal/cache/netmap.go b/internal/cache/netmap.go index 6d91fe7..ce01b47 100644 --- a/internal/cache/netmap.go +++ b/internal/cache/netmap.go @@ -53,7 +53,7 @@ func (c *NetmapCache) Get() *netmap.NetMap { result, ok := entry.(netmap.NetMap) if !ok { c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), - zap.String("expected", fmt.Sprintf("%T", result))) + zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath)) return nil } diff --git a/internal/handler/browse.go b/internal/handler/browse.go index 64ad1f5..2d0e34d 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -230,7 +230,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck } for objExt := range resp { if objExt.Error != nil { - log.Error(logs.FailedToHeadObject, zap.Error(objExt.Error)) + log.Error(logs.FailedToHeadObject, zap.Error(objExt.Error), logs.TagField(logs.TagExternalStorage)) result.hasErrors = true continue } @@ -273,7 +273,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re }) if err != nil { wg.Done() - log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err)) + log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath)) } select { case <-ctx.Done(): @@ -283,7 +283,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re } }) if err != nil { - log.Error(logs.FailedToIterateOverResponse, zap.Error(err)) + log.Error(logs.FailedToIterateOverResponse, zap.Error(err), logs.TagField(logs.TagDatapath)) } wg.Wait() }() diff --git a/internal/handler/download.go b/internal/handler/download.go index d5fac23..783fe09 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -43,6 +43,8 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + log.Error(logs.FailedToCheckIfSettingsNodeExist, zap.String("cid", bktInfo.CID.String()), + zap.Error(checkS3Err), logs.TagField(logs.TagExternalStorageTree)) logAndSendBucketError(c, log, checkS3Err) return } @@ -121,13 +123,13 @@ func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, res }), ) if errIter != nil { - log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) + log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) return } else if objectsWritten == 0 { - log.Warn(logs.ObjectsNotFound) + log.Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) } if err := zipWriter.Close(); err != nil { - log.Error(logs.CloseZipWriter, zap.Error(err)) + log.Error(logs.CloseZipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } } } @@ -187,10 +189,10 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res defer func() { if err := tarWriter.Close(); err != nil { - log.Error(logs.CloseTarWriter, zap.Error(err)) + log.Error(logs.CloseTarWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } if err := gzipWriter.Close(); err != nil { - log.Error(logs.CloseGzipWriter, zap.Error(err)) + log.Error(logs.CloseGzipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } }() @@ -204,9 +206,9 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res }), ) if errIter != nil { - log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) + log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) } else if objectsWritten == 0 { - log.Warn(logs.ObjectsNotFound) + log.Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) } } } @@ -237,18 +239,18 @@ func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID resGet, err := h.frostfs.GetObject(ctx, prm) if err != nil { - log.Error(logs.FailedToGetObject, zap.Error(err)) + log.Error(logs.FailedToGetObject, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return false } fileWriter, err := createArchiveHeader(&resGet.Header) if err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.Error(err)) + log.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) return false } if err = writeToArchive(resGet, fileWriter, buf); err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.Error(err)) + log.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) return false } @@ -264,7 +266,8 @@ func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, prefix, err := url.QueryUnescape(prefix) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), + zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) return nil, err } @@ -273,7 +276,7 @@ func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, resSearch, err := h.search(ctx, cnrID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage)) ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return nil, err } diff --git a/internal/handler/filter.go b/internal/handler/filter.go index 745718a..da99db7 100644 --- a/internal/handler/filter.go +++ b/internal/handler/filter.go @@ -50,7 +50,8 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st l.Debug(logs.AddAttributeToResultObject, zap.String("key", k), - zap.String("val", v)) + zap.String("val", v), + logs.TagField(logs.TagDatapath)) }) return result, err diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4e23c3b..b27c607 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -206,11 +206,13 @@ func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) if err != nil { + log.Error(logs.FailedToGetLatestVersionOfObject, zap.Error(err), zap.String("cid", cnrID.String()), + zap.String("path", path), logs.TagField(logs.TagExternalStorageTree)) logAndSendBucketError(c, log, err) return } if foundOID.IsDeleteMarker { - log.Error(logs.ObjectWasDeleted) + log.Error(logs.ObjectWasDeleted, logs.TagField(logs.TagExternalStorageTree)) ResponseError(c, "object deleted", fasthttp.StatusNotFound) return } @@ -230,14 +232,16 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte key, err := url.QueryUnescape(key) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), + zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) return } val, err = url.QueryUnescape(val) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), + zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -271,7 +275,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { res, err := h.search(ctx, cnrID, attrKey, attrVal, object.MatchStringEqual) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("could not search for objects: %w", err) } defer res.Close() @@ -282,13 +286,13 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cn if n == 0 { switch { case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): - log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName) + log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal) case errors.Is(err, io.EOF): - log.Error(logs.ObjectNotFound, zap.Error(err)) + log.Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("object not found: %w", err) default: - log.Error(logs.ReadObjectListFailed, zap.Error(err)) + log.Error(logs.ReadObjectListFailed, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("read object list failed: %w", err) } } @@ -330,11 +334,16 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log * cnrID, err := h.resolveContainer(ctx, containerName) if err != nil { + log.Error(logs.CouldNotResolveContainerID, zap.Error(err), zap.String("cnrName", containerName), + logs.TagField(logs.TagDatapath)) return nil, err } bktInfo, err := h.readContainer(ctx, *cnrID) if err != nil { + log.Error(logs.CouldNotGetContainerInfo, zap.Error(err), zap.String("cnrName", containerName), + zap.String("cnrName", cnrID.String()), + logs.TagField(logs.TagExternalStorage)) return nil, err } @@ -342,7 +351,8 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log * log.Warn(logs.CouldntPutBucketIntoCache, zap.String("bucket name", bktInfo.Name), zap.Stringer("bucket cid", bktInfo.CID), - zap.Error(err)) + zap.Error(err), + logs.TagField(logs.TagDatapath)) } return bktInfo, nil diff --git a/internal/handler/head.go b/internal/handler/head.go index 94f5ccb..0b5adc4 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -67,7 +67,8 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid req.log.Info(logs.CouldntParseCreationDate, zap.String("key", key), zap.String("val", val), - zap.Error(err)) + zap.Error(err), + logs.TagField(logs.TagDatapath)) continue } req.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat)) @@ -131,6 +132,8 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { } checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + log.Error(logs.FailedToCheckIfSettingsNodeExist, zap.String("cid", bktInfo.CID.String()), + zap.Error(checkS3Err), logs.TagField(logs.TagExternalStorageTree)) logAndSendBucketError(c, log, checkS3Err) return } @@ -144,7 +147,6 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) } else { logAndSendBucketError(c, log, checkS3Err) - return } } diff --git a/internal/handler/multipart.go b/internal/handler/multipart.go index ebf5edd..5ed2350 100644 --- a/internal/handler/multipart.go +++ b/internal/handler/multipart.go @@ -33,7 +33,7 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF name := part.FormName() if name == "" { - l.Debug(logs.IgnorePartEmptyFormName) + l.Debug(logs.IgnorePartEmptyFormName, logs.TagField(logs.TagDatapath)) continue } @@ -41,9 +41,9 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF // ignore multipart/form-data values if filename == "" { - l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name)) + l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name), logs.TagField(logs.TagDatapath)) if err = part.Close(); err != nil { - l.Warn(logs.FailedToCloseReader, zap.Error(err)) + l.Warn(logs.FailedToCloseReader, zap.Error(err), logs.TagField(logs.TagDatapath)) } continue } diff --git a/internal/handler/multipart_test.go b/internal/handler/multipart_test.go index 2c50a87..431d0d6 100644 --- a/internal/handler/multipart_test.go +++ b/internal/handler/multipart_test.go @@ -112,7 +112,7 @@ func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (Mul name := part.FormName() if name == "" { - l.Debug(logs.IgnorePartEmptyFormName) + l.Debug(logs.IgnorePartEmptyFormName, logs.TagField(logs.TagDatapath)) continue } @@ -120,8 +120,7 @@ func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (Mul // ignore multipart/form-data values if filename == "" { - l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name)) - + l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name), logs.TagField(logs.TagDatapath)) continue } diff --git a/internal/handler/reader.go b/internal/handler/reader.go index cbd8294..e8ac098 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -110,7 +110,8 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A if err = req.setTimestamp(val); err != nil { req.log.Error(logs.CouldntParseCreationDate, zap.String("val", val), - zap.Error(err)) + zap.Error(err), + logs.TagField(logs.TagDatapath)) } case object.AttributeContentType: contentType = val @@ -144,7 +145,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A return payload, nil }, filename) if err != nil && err != io.EOF { - req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) + req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index d1953c9..272e911 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -68,14 +68,14 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { - log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) + log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } filtered, err := filterHeaders(log, &c.Request.Header) if err != nil { - log.Error(logs.FailedToFilterHeaders, zap.Error(err)) + log.Error(logs.FailedToFilterHeaders, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, err.Error(), fasthttp.StatusBadRequest) return } @@ -106,7 +106,7 @@ func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file Mul attributes, err := h.extractAttributes(c, log, filtered) if err != nil { - log.Error(logs.FailedToGetAttributes, zap.Error(err)) + log.Error(logs.FailedToGetAttributes, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -119,13 +119,14 @@ func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file Mul log.Debug(logs.ObjectUploaded, zap.String("oid", idObj.EncodeToString()), zap.String("FileName", file.FileName()), + logs.TagField(logs.TagExternalStorage), ) addr := newAddress(bkt.CID, idObj) c.Response.Header.SetContentType(jsonHeader) // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { - log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) + log.Error(logs.CouldNotEncodeResponse, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) return } @@ -162,13 +163,14 @@ func (h *Handler) extractAttributes(c *fasthttp.RequestCtx, log *zap.Logger, fil now := time.Now() if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { - log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err)) + log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err), + logs.TagField(logs.TagDatapath)) } else { now = parsed } } if err := utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { - log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) + log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err), logs.TagField(logs.TagDatapath)) return nil, err } attributes := make([]object.Attribute, 0, len(filtered)) @@ -205,7 +207,7 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read commonAttributes, err := h.extractAttributes(c, log, filtered) if err != nil { - log.Error(logs.FailedToGetAttributes, zap.Error(err)) + log.Error(logs.FailedToGetAttributes, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -213,16 +215,16 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read reader := file if bytes.EqualFold(c.Request.Header.Peek(fasthttp.HeaderContentEncoding), []byte("gzip")) { - log.Debug(logs.GzipReaderSelected) + log.Debug(logs.GzipReaderSelected, logs.TagField(logs.TagDatapath)) gzipReader, err := gzip.NewReader(file) if err != nil { - log.Error(logs.FailedToCreateGzipReader, zap.Error(err)) + log.Error(logs.FailedToCreateGzipReader, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could read gzip file: "+err.Error(), fasthttp.StatusBadRequest) return } defer func() { if err := gzipReader.Close(); err != nil { - log.Warn(logs.FailedToCloseReader, zap.Error(err)) + log.Warn(logs.FailedToCloseReader, zap.Error(err), logs.TagField(logs.TagDatapath)) } }() reader = gzipReader @@ -234,7 +236,7 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read if errors.Is(err, io.EOF) { break } else if err != nil { - log.Error(logs.FailedToReadFileFromTar, zap.Error(err)) + log.Error(logs.FailedToReadFileFromTar, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not get next entry: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -258,6 +260,7 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read log.Debug(logs.ObjectUploaded, zap.String("oid", idObj.EncodeToString()), zap.String("FileName", fileName), + logs.TagField(logs.TagExternalStorage), ) } } @@ -266,7 +269,7 @@ func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *za statusCode, msg, additionalFields := formErrorResponse("could not store file in frostfs", err) logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) - log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) + log.Error(logs.CouldNotStoreFileInFrostfs, append(logFields, logs.TagField(logs.TagExternalStorage))...) ResponseError(r, msg, statusCode) } diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 74932f3..0a1dc62 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -39,7 +39,7 @@ func (r *request) handleFrostFSErr(err error, start time.Time) { statusCode, msg, additionalFields := formErrorResponse("could not receive object", err) logFields = append(logFields, additionalFields...) - r.log.Error(logs.CouldNotReceiveObject, logFields...) + r.log.Error(logs.CouldNotReceiveObject, append(logFields, logs.TagField(logs.TagExternalStorage))...) ResponseError(r.RequestCtx, msg, statusCode) } @@ -85,7 +85,7 @@ func isValidValue(s string) bool { } func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { - log.Error(logs.CouldntGetBucket, zap.Error(err)) + log.Error(logs.CouldNotGetBucket, zap.Error(err), logs.TagField(logs.TagDatapath)) if client.IsErrContainerNotFound(err) { ResponseError(c, "Not Found", fasthttp.StatusNotFound) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 5f60b9b..072e9c9 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -1,63 +1,43 @@ package logs +import "go.uber.org/zap" + +const ( + TagFieldName = "tag" + + TagApp = "app" + TagDatapath = "datapath" + TagExternalStorage = "external_storage" + TagExternalStorageTree = "external_storage_tree" +) + +func TagField(tag string) zap.Field { + return zap.String(TagFieldName, tag) +} + +// Log messages with the "app" tag. const ( - CouldntParseCreationDate = "couldn't parse creation date" - CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" - CouldNotReceiveObject = "could not receive object" - ObjectWasDeleted = "object was deleted" - CouldNotSearchForObjects = "could not search for objects" - ObjectNotFound = "object not found" - ReadObjectListFailed = "read object list failed" - FailedToAddObjectToArchive = "failed to add object to archive" - FailedToGetObject = "failed to get object" - IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" - ObjectsNotFound = "objects not found" - CloseZipWriter = "close zip writer" ServiceIsRunning = "service is running" ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" ShuttingDownService = "shutting down service" CantShutDownService = "can't shut down service" CantGracefullyShutDownService = "can't gracefully shut down service, force stop" - IgnorePartEmptyFormName = "ignore part, empty form name" - IgnorePartEmptyFilename = "ignore part, empty filename" - CouldNotReceiveMultipartForm = "could not receive multipart/form" - CouldNotParseClientTime = "could not parse client time" - CouldNotPrepareExpirationHeader = "could not prepare expiration header" - CouldNotEncodeResponse = "could not encode response" - CouldNotStoreFileInFrostfs = "could not store file in frostfs" - AddAttributeToResultObject = "add attribute to result object" FailedToCreateResolver = "failed to create resolver" FailedToCreateWorkerPool = "failed to create worker pool" - FailedToReadIndexPageTemplate = "failed to read index page template" - SetCustomIndexPageTemplate = "set custom index page template" - ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" - MetricsAreDisabled = "metrics are disabled" - NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" StartingApplication = "starting application" StartingServer = "starting server" ListenAndServe = "listen and serve" ShuttingDownWebServer = "shutting down web server" FailedToShutdownTracing = "failed to shutdown tracing" - SIGHUPConfigReloadStarted = "SIGHUP config reload started" - FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" - FailedToReloadConfig = "failed to reload config" - LogLevelWontBeUpdated = "log level won't be updated" - FailedToUpdateResolvers = "failed to update resolvers" - FailedToReloadServerParameters = "failed to reload server parameters" - SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" AddedPathUploadCid = "added path /upload/{cid}" AddedPathGetCidOid = "added path /get/{cid}/{oid}" AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" - Request = "request" - CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" FailedToAddServer = "failed to add server" AddServer = "add server" NoHealthyServers = "no healthy servers" FailedToInitializeTracing = "failed to initialize tracing" - TracingConfigUpdated = "tracing config updated" - ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" @@ -66,33 +46,86 @@ const ( FailedToDialConnectionPool = "failed to dial connection pool" FailedToCreateTreePool = "failed to create tree pool" FailedToDialTreePool = "failed to dial tree pool" - AddedStoragePeer = "added storage peer" - CouldntGetBucket = "could not get bucket" - CouldntPutBucketIntoCache = "couldn't put bucket info into cache" - FailedToSumbitTaskToPool = "failed to submit task to pool" - FailedToHeadObject = "failed to head object" - FailedToIterateOverResponse = "failed to iterate over search response" - InvalidCacheEntryType = "invalid cache entry type" - InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" - InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" - FailedToUnescapeQuery = "failed to unescape query" ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" - WarnDuplicateAddress = "duplicate address" + FailedToSumbitTaskToPool = "failed to submit task to pool" MultinetDialSuccess = "multinet dial successful" MultinetDialFail = "multinet dial failed" + ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" + MetricsAreDisabled = "metrics are disabled" + NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" + SIGHUPConfigReloadStarted = "SIGHUP config reload started" + FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" + FailedToReloadConfig = "failed to reload config" + FailedToUpdateResolvers = "failed to update resolvers" + FailedToReloadServerParameters = "failed to reload server parameters" + SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" + TracingConfigUpdated = "tracing config updated" + ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" + AddedStoragePeer = "added storage peer" + InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" + InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" + WarnDuplicateAddress = "duplicate address" FailedToLoadMultinetConfig = "failed to load multinet config" MultinetConfigWontBeUpdated = "multinet config won't be updated" - ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" - CouldntCacheNetmap = "couldn't cache netmap" - FailedToFilterHeaders = "failed to filter headers" - FailedToReadFileFromTar = "failed to read file from tar" - FailedToGetAttributes = "failed to get attributes" - ObjectUploaded = "object uploaded" - CloseGzipWriter = "close gzip writer" - CloseTarWriter = "close tar writer" - FailedToCloseReader = "failed to close reader" - FailedToCreateGzipReader = "failed to create gzip reader" - GzipReaderSelected = "gzip reader selected" + LogLevelWontBeUpdated = "log level won't be updated" + TagsLogConfigWontBeUpdated = "tags log config won't be updated" + FailedToReadIndexPageTemplate = "failed to read index page template" + SetCustomIndexPageTemplate = "set custom index page template" +) + +// Log messages with the "datapath" tag. +const ( + CouldntParseCreationDate = "couldn't parse creation date" + CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" + FailedToAddObjectToArchive = "failed to add object to archive" + CloseZipWriter = "close zip writer" + IgnorePartEmptyFormName = "ignore part, empty form name" + IgnorePartEmptyFilename = "ignore part, empty filename" + CouldNotParseClientTime = "could not parse client time" + CouldNotPrepareExpirationHeader = "could not prepare expiration header" + CouldNotEncodeResponse = "could not encode response" + AddAttributeToResultObject = "add attribute to result object" + Request = "request" + CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" + CouldntPutBucketIntoCache = "couldn't put bucket info into cache" + FailedToIterateOverResponse = "failed to iterate over search response" + InvalidCacheEntryType = "invalid cache entry type" + FailedToUnescapeQuery = "failed to unescape query" + CouldntCacheNetmap = "couldn't cache netmap" + FailedToCloseReader = "failed to close reader" + FailedToFilterHeaders = "failed to filter headers" + FailedToReadFileFromTar = "failed to read file from tar" + FailedToGetAttributes = "failed to get attributes" + CloseGzipWriter = "close gzip writer" + CloseTarWriter = "close tar writer" + FailedToCreateGzipReader = "failed to create gzip reader" + GzipReaderSelected = "gzip reader selected" + CouldNotReceiveMultipartForm = "could not receive multipart/form" + ObjectsNotFound = "objects not found" + IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" + CouldNotGetBucket = "could not get bucket" + CouldNotResolveContainerID = "could not resolve container id" +) + +// Log messages with the "external_storage" tag. +const ( + CouldNotReceiveObject = "could not receive object" + CouldNotSearchForObjects = "could not search for objects" + ObjectNotFound = "object not found" + ReadObjectListFailed = "read object list failed" + CouldNotStoreFileInFrostfs = "could not store file in frostfs" + FailedToHeadObject = "failed to head object" + ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" + FailedToGetObject = "failed to get object" + ObjectUploaded = "object uploaded" + CouldNotGetContainerInfo = "could not get container info" +) + +// Log messages with the "external_storage_tree" tag. +const ( + ObjectWasDeleted = "object was deleted" + FailedToGetLatestVersionOfObject = "failed to get latest version of object" + FailedToCheckIfSettingsNodeExist = "Failed to check if settings node exists" ) diff --git a/internal/net/event_handler.go b/internal/net/event_handler.go index 9520c01..2826d35 100644 --- a/internal/net/event_handler.go +++ b/internal/net/event_handler.go @@ -17,9 +17,11 @@ func (l LogEventHandler) DialPerformed(sourceIP net.Addr, _, address string, err sourceIPString = sourceIP.Network() + "://" + sourceIP.String() } if err == nil { - l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString), zap.String("destination", address)) + l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString), + zap.String("destination", address), logs.TagField(logs.TagApp)) } else { - l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString), zap.String("destination", address), zap.Error(err)) + l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString), + zap.String("destination", address), logs.TagField(logs.TagApp)) } } diff --git a/internal/service/frostfs/source.go b/internal/service/frostfs/source.go index de6c681..84f7b74 100644 --- a/internal/service/frostfs/source.go +++ b/internal/service/frostfs/source.go @@ -40,7 +40,7 @@ func (s *Source) NetMapSnapshot(ctx context.Context) (netmap.NetMap, error) { } if err = s.netmapCache.Put(netmapSnapshot); err != nil { - s.log.Warn(logs.CouldntCacheNetmap, zap.Error(err)) + s.log.Warn(logs.CouldntCacheNetmap, zap.Error(err), logs.TagField(logs.TagDatapath)) } return netmapSnapshot, nil diff --git a/metrics/service.go b/metrics/service.go index dea5ac0..e6b803b 100644 --- a/metrics/service.go +++ b/metrics/service.go @@ -25,24 +25,24 @@ type Config struct { // Start runs http service with the exposed endpoint on the configured port. func (ms *Service) Start() { if ms.enabled { - ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr)) + ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp)) err := ms.ListenAndServe() if err != nil && err != http.ErrServerClosed { - ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort) + ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort, logs.TagField(logs.TagApp)) } } else { - ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled) + ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled, logs.TagField(logs.TagApp)) } } // ShutDown stops the service. func (ms *Service) ShutDown(ctx context.Context) { - ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr)) + ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp)) err := ms.Shutdown(ctx) if err != nil { - ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err)) + ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err), logs.TagField(logs.TagApp)) if err = ms.Close(); err != nil { - ms.log.Panic(logs.CantShutDownService, zap.Error(err)) + ms.log.Panic(logs.CantShutDownService, zap.Error(err), logs.TagField(logs.TagApp)) } } } From 1e8fa19bb9ec13bd4184801ecae28983943f18ca Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 7 Feb 2025 13:55:17 +0300 Subject: [PATCH 519/548] [#195] Make all initial logging tags as default tags Signed-off-by: Alex Vanin --- cmd/http-gw/settings.go | 2 +- docs/gate-configuration.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index d8331ad..12d73d6 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -196,7 +196,7 @@ var ignore = map[string]struct{}{ cmdVersion: {}, } -var defaultTags = []string{logs.TagApp, logs.TagDatapath} +var defaultTags = []string{logs.TagApp, logs.TagDatapath, logs.TagExternalStorage, logs.TagExternalStorageTree} type Logger struct { logger *zap.Logger diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 4f9bc3b..191e9bb 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -211,8 +211,8 @@ tags: * `app` - common application logs (enabled by default). * `datapath` - main logic of application (enabled by default). -* `external_storage` - external interaction with storage node. -* `external_storage_tree` - external interaction with tree service in storage node. +* `external_storage` - external interaction with storage node (enabled by default). +* `external_storage_tree` - external interaction with tree service in storage node (enabled by default). # `web` section From c509ce0b28748e79c2442b5ef2e434232b84967d Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 7 Feb 2025 13:56:11 +0300 Subject: [PATCH 520/548] [#195] Fix log record grouping Signed-off-by: Alex Vanin --- internal/logs/logs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 072e9c9..f8f1da9 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -49,7 +49,6 @@ const ( ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" - FailedToSumbitTaskToPool = "failed to submit task to pool" MultinetDialSuccess = "multinet dial successful" MultinetDialFail = "multinet dial failed" ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" @@ -107,6 +106,7 @@ const ( IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" CouldNotGetBucket = "could not get bucket" CouldNotResolveContainerID = "could not resolve container id" + FailedToSumbitTaskToPool = "failed to submit task to pool" ) // Log messages with the "external_storage" tag. From 11846df2660e782f414bfd10edd3bdc61e2308b1 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 05:48:14 +0300 Subject: [PATCH 521/548] [#145] handler: Add spans to detail the trace Signed-off-by: Roman Loginov --- internal/handler/download.go | 20 +++++++++++++++++--- internal/handler/handler.go | 12 +++++++++++- internal/handler/head.go | 9 ++++++++- internal/handler/upload.go | 18 ++++++++++++++++-- 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/internal/handler/download.go b/internal/handler/download.go index 783fe09..b398a54 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -16,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -25,11 +26,14 @@ import ( // DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadByAddressOrBucketName") + defer span.End() + utils.SetContextToRequest(ctx, c) + cidParam := c.UserValue("cid").(string) oidParam := c.UserValue("oid").(string) downloadParam := c.QueryArgs().GetBool("download") - ctx := utils.GetContextFromRequest(c) log := utils.GetReqLogOrDefault(ctx, h.log).With( zap.String("cid", cidParam), zap.String("oid", oidParam), @@ -67,6 +71,10 @@ func shouldDownload(oidParam string, downloadParam bool) bool { // DownloadByAttribute handles attribute-based download requests. func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadByAttribute") + defer span.End() + utils.SetContextToRequest(ctx, c) + h.byAttribute(c, h.receiveFile) } @@ -88,9 +96,12 @@ func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op // DownloadZip handles zip by prefix requests. func (h *Handler) DownloadZip(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadZip") + defer span.End() + utils.SetContextToRequest(ctx, c) + scid, _ := c.UserValue("cid").(string) - ctx := utils.GetContextFromRequest(c) log := utils.GetReqLogOrDefault(ctx, h.log) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { @@ -154,9 +165,12 @@ func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, // DownloadTar forms tar.gz from objects by prefix. func (h *Handler) DownloadTar(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadTar") + defer span.End() + utils.SetContextToRequest(ctx, c) + scid, _ := c.UserValue("cid").(string) - ctx := utils.GetContextFromRequest(c) log := utils.GetReqLogOrDefault(ctx, h.log) bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index b27c607..69aecbf 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" @@ -195,6 +196,9 @@ func New(params *AppParams, config Config, tree layer.TreeService, workerPool *a // byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID, objID oid.ID, handler func(context.Context, request, oid.Address)) { + ctx, span := tracing.StartSpanFromContext(ctx, "handler.byNativeAddress") + defer span.End() + addr := newAddress(cnrID, objID) handler(ctx, req, addr) } @@ -202,6 +206,9 @@ func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID // byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that // resolves object address from S3-like path /. func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path string, handler func(context.Context, request, oid.Address)) { + ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path") + defer span.End() + c, log := req.RequestCtx, req.log foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) @@ -382,6 +389,10 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket } func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.browseIndex") + defer span.End() + utils.SetContextToRequest(ctx, c) + if !h.config.IndexPageEnabled() { c.SetStatusCode(fasthttp.StatusNotFound) return @@ -390,7 +401,6 @@ func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) { cidURLParam := c.UserValue("cid").(string) oidURLParam := c.UserValue("oid").(string) - ctx := utils.GetContextFromRequest(c) reqLog := utils.GetReqLogOrDefault(ctx, h.log) log := reqLog.With(zap.String("cid", cidURLParam), zap.String("oid", oidURLParam)) diff --git a/internal/handler/head.go b/internal/handler/head.go index 0b5adc4..7718c9c 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/valyala/fasthttp" @@ -116,10 +117,12 @@ func idsToResponse(resp *fasthttp.Response, obj *object.Object) { // HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format. func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.HeadByAddressOrBucketName") + defer span.End() + cidParam, _ := c.UserValue("cid").(string) oidParam, _ := c.UserValue("oid").(string) - ctx := utils.GetContextFromRequest(c) log := utils.GetReqLogOrDefault(ctx, h.log).With( zap.String("cid", cidParam), zap.String("oid", oidParam), @@ -152,5 +155,9 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { // HeadByAttribute handles attribute-based head requests. func (h *Handler) HeadByAttribute(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.HeadByAttribute") + defer span.End() + utils.SetContextToRequest(ctx, c) + h.byAttribute(c, h.headObject) } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 272e911..48d0495 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -17,6 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -50,13 +51,16 @@ func (pr *putResponse) encode(w io.Writer) error { // Upload handles multipart upload request. func (h *Handler) Upload(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.Upload") + defer span.End() + utils.SetContextToRequest(ctx, c) + var file MultipartFile scid, _ := c.UserValue("cid").(string) bodyStream := c.RequestBodyStream() drainBuf := make([]byte, drainBufSize) - ctx := utils.GetContextFromRequest(c) reqLog := utils.GetReqLogOrDefault(ctx, h.log) log := reqLog.With(zap.String("cid", scid)) @@ -102,6 +106,11 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file MultipartFile, filtered map[string]string) { c, log := req.RequestCtx, req.log + + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.uploadSingleObject") + defer span.End() + utils.SetContextToRequest(ctx, c) + setIfNotExist(filtered, object.AttributeFileName, file.FileName()) attributes, err := h.extractAttributes(c, log, filtered) @@ -160,6 +169,7 @@ func (h *Handler) uploadObject(c *fasthttp.RequestCtx, bkt *data.BucketInfo, att } func (h *Handler) extractAttributes(c *fasthttp.RequestCtx, log *zap.Logger, filtered map[string]string) ([]object.Attribute, error) { + ctx := utils.GetContextFromRequest(c) now := time.Now() if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { @@ -169,7 +179,7 @@ func (h *Handler) extractAttributes(c *fasthttp.RequestCtx, log *zap.Logger, fil now = parsed } } - if err := utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { + if err := utils.PrepareExpirationHeader(ctx, h.frostfs, filtered, now); err != nil { log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err), logs.TagField(logs.TagDatapath)) return nil, err } @@ -200,6 +210,10 @@ func newAttribute(key string, val string) object.Attribute { func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.ReadCloser, filtered map[string]string) { c, log := req.RequestCtx, req.log + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.explodeArchive") + defer span.End() + utils.SetContextToRequest(ctx, c) + // remove user attributes which vary for each file in archive // to guarantee that they won't appear twice delete(filtered, object.AttributeFileName) From bfe24a458b429a38b98b77c2b08a059d3b1e7246 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 05:48:35 +0300 Subject: [PATCH 522/548] [#145] frostfs: Add spans to detail the trace Signed-off-by: Roman Loginov --- internal/service/frostfs/frostfs.go | 28 +++++++++++++++++++ .../service/frostfs/multi_object_reader.go | 7 +++++ internal/service/frostfs/tree_pool_wrapper.go | 7 +++++ 3 files changed, 42 insertions(+) diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index c6af526..4cf45a4 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -35,6 +36,9 @@ func NewFrostFS(p *pool.Pool) *FrostFS { // Container implements frostfs.FrostFS interface method. func (x *FrostFS) Container(ctx context.Context, containerPrm handler.PrmContainer) (*container.Container, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.Container") + defer span.End() + prm := pool.PrmContainerGet{ ContainerID: containerPrm.ContainerID, } @@ -49,6 +53,9 @@ func (x *FrostFS) Container(ctx context.Context, containerPrm handler.PrmContain // CreateObject implements frostfs.FrostFS interface method. func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) (oid.ID, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.CreateObject") + defer span.End() + var prmPut pool.PrmObjectPut prmPut.SetHeader(*prm.Object) prmPut.SetPayload(prm.Payload) @@ -83,6 +90,9 @@ func (x payloadReader) Read(p []byte) (int, error) { // HeadObject implements frostfs.FrostFS interface method. func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*object.Object, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.HeadObject") + defer span.End() + var prmHead pool.PrmObjectHead prmHead.SetAddress(prm.Address) @@ -100,6 +110,9 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o // GetObject implements frostfs.FrostFS interface method. func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*handler.Object, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.GetObject") + defer span.End() + var prmGet pool.PrmObjectGet prmGet.SetAddress(prm.Address) @@ -120,6 +133,9 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han // RangeObject implements frostfs.FrostFS interface method. func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) (io.ReadCloser, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.RangeObject") + defer span.End() + var prmRange pool.PrmObjectRange prmRange.SetAddress(prm.Address) prmRange.SetOffset(prm.PayloadRange[0]) @@ -139,6 +155,9 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) ( // SearchObjects implements frostfs.FrostFS interface method. func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch) (handler.ResObjectSearch, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.SearchObjects") + defer span.End() + var prmSearch pool.PrmObjectSearch prmSearch.SetContainerID(prm.Container) prmSearch.SetFilters(prm.Filters) @@ -157,6 +176,9 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch // GetEpochDurations implements frostfs.FrostFS interface method. func (x *FrostFS) GetEpochDurations(ctx context.Context) (*utils.EpochDurations, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.GetEpochDurations") + defer span.End() + networkInfo, err := x.pool.NetworkInfo(ctx) if err != nil { return nil, err @@ -175,6 +197,9 @@ func (x *FrostFS) GetEpochDurations(ctx context.Context) (*utils.EpochDurations, } func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.NetmapSnapshot") + defer span.End() + netmapSnapshot, err := x.pool.NetMapSnapshot(ctx) if err != nil { return netmapSnapshot, handleObjectError("get netmap via connection pool", err) @@ -196,6 +221,9 @@ func NewResolverFrostFS(p *pool.Pool) *ResolverFrostFS { // SystemDNS implements resolver.FrostFS interface method. func (x *ResolverFrostFS) SystemDNS(ctx context.Context) (string, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.SystemDNS") + defer span.End() + networkInfo, err := x.pool.NetworkInfo(ctx) if err != nil { return "", handleObjectError("read network info via client", err) diff --git a/internal/service/frostfs/multi_object_reader.go b/internal/service/frostfs/multi_object_reader.go index 93f1f60..b943474 100644 --- a/internal/service/frostfs/multi_object_reader.go +++ b/internal/service/frostfs/multi_object_reader.go @@ -9,6 +9,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -74,6 +75,9 @@ var ( ) func (x *FrostFS) InitMultiObjectReader(ctx context.Context, p handler.PrmInitMultiObjectReader) (io.Reader, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.InitMultiObjectReader") + defer span.End() + combinedObj, err := x.GetObject(ctx, handler.PrmObjectGet{ PrmAuth: handler.PrmAuth{BearerToken: p.Bearer}, Address: p.Addr, @@ -215,6 +219,9 @@ func (x *MultiObjectReader) Read(p []byte) (n int, err error) { // InitFrostFSObjectPayloadReader initializes payload reader of the FrostFS object. // Zero range corresponds to full payload (panics if only offset is set). func (x *FrostFS) InitFrostFSObjectPayloadReader(ctx context.Context, p GetFrostFSParams) (io.ReadCloser, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.InitFrostFSObjectPayloadReader") + defer span.End() + var prmAuth handler.PrmAuth if p.Off+p.Ln != 0 { diff --git a/internal/service/frostfs/tree_pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go index f6be05f..410acda 100644 --- a/internal/service/frostfs/tree_pool_wrapper.go +++ b/internal/service/frostfs/tree_pool_wrapper.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apitree "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/tree" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" ) @@ -46,6 +47,9 @@ func NewPoolWrapper(p *treepool.Pool) *PoolWrapper { } func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]tree.NodeResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.GetNodes") + defer span.End() + poolPrm := treepool.GetNodesParams{ CID: prm.CnrID, TreeID: prm.TreeID, @@ -93,6 +97,9 @@ func handleError(err error) error { } func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.GetSubTree") + defer span.End() + order := treepool.NoneOrder if sort { order = treepool.AscendingOrder From 412886c24fc002e6287d8ff659ecac5f749835bb Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 05:48:46 +0300 Subject: [PATCH 523/548] [#145] tree: Add spans to detail the trace Signed-off-by: Roman Loginov --- tree/tree.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tree/tree.go b/tree/tree.go index bf0aff9..315e5ad 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -190,6 +191,9 @@ func (m *multiSystemNode) Old() []*treeNode { } func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetLatestVersion") + defer span.End() + nodes, err := c.GetVersions(ctx, cnrID, objectName) if err != nil { return nil, err @@ -204,6 +208,9 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s } func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]NodeResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetVersions") + defer span.End() + meta := []string{oidKV, isDeleteMarkerKV, sizeKV} path := pathFromName(objectName) @@ -220,6 +227,9 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string } func (c *Tree) CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.CheckSettingsNodeExists") + defer span.End() + _, err := c.getSystemNode(ctx, bktInfo, settingsFileName) if err != nil { return err @@ -308,6 +318,9 @@ func pathFromName(objectName string) []string { } func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix") + defer span.End() + rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix) if err != nil { return nil, "", err From 20319418cc33a68a3698deab00b69ef1a13d7ff5 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 11:21:53 +0300 Subject: [PATCH 524/548] [#145] Update frostfs-observability version The new version of frostfs-observability has improved the detail of tracing low-level rpc calls by adding send and receive events. Signed-off-by: Roman Loginov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d0bfcbb..b3f590e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.22 require ( - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 diff --git a/go.sum b/go.sum index 8eb4ea9..9ee2d9c 100644 --- a/go.sum +++ b/go.sum @@ -42,10 +42,10 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 h1:/Z8DfbLZXp7exUQWUKoG/9tbFdI9d5lV1qSReaYoG8I= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 47d74a5a77d5fa89955f2345c3b417c72aee1791 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Tue, 4 Feb 2025 13:21:57 +0300 Subject: [PATCH 525/548] [#174] Add slash clipping for FileName attribute According to the FrostFS API specification, the FileName attribute cannot contain a slash at the beginning. Signed-off-by: Roman Loginov --- CHANGELOG.md | 1 + internal/handler/handler.go | 14 +++++++++++++- internal/handler/handler_test.go | 32 ++++++++++++++++++++++++++++++-- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30fcf7a..51080bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This document outlines major changes between releases. ### Added - Add handling quota limit reached error (#187) +- Add slash clipping for FileName attribute (#174) ## [0.32.2] - 2025-02-03 diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 69aecbf..532cdc4 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -253,6 +253,10 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte return } + if key == attrFileName { + val = prepareFileName(val) + } + log = log.With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val)) bktInfo, err := h.getBucketInfo(ctx, cidParam, log) @@ -294,7 +298,7 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cn switch { case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) - return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal) + return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, prepareFileName(attrVal)) case errors.Is(err, io.EOF): log.Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("object not found: %w", err) @@ -315,6 +319,14 @@ func (h *Handler) needSearchByFileName(key, val string) bool { return strings.HasPrefix(val, "/") && strings.Count(val, "/") == 1 || !strings.Contains(val, "/") } +func prepareFileName(fileName string) string { + if strings.HasPrefix(fileName, "/") { + return fileName[1:] + } + + return fileName +} + // resolveContainer decode container id, if it's not a valid container id // then trey to resolve name using provided resolver. func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 53c9739..1638f9f 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -239,6 +239,10 @@ func TestBasic(t *testing.T) { r = prepareGetByAttributeRequest(ctx, bktName, keyAttr, valAttr) hc.Handler().DownloadByAttribute(r) require.Equal(t, content, string(r.Response.Body())) + + r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, "/"+objFileName) + hc.Handler().DownloadByAttribute(r) + require.Equal(t, content, string(r.Response.Body())) }) t.Run("head by attribute", func(t *testing.T) { @@ -246,6 +250,11 @@ func TestBasic(t *testing.T) { hc.Handler().HeadByAttribute(r) require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + + r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, "/"+objFileName) + hc.Handler().HeadByAttribute(r) + require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) + require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) }) t.Run("zip", func(t *testing.T) { @@ -293,8 +302,8 @@ func TestFindObjectByAttribute(t *testing.T) { err = json.Unmarshal(r.Response.Body(), &putRes) require.NoError(t, err) - testAttrVal1 := "test-attr-val1" - testAttrVal2 := "test-attr-val2" + testAttrVal1 := "/folder/cat.jpg" + testAttrVal2 := "cat.jpg" testAttrVal3 := "test-attr-val3" for _, tc := range []struct { @@ -340,6 +349,14 @@ func TestFindObjectByAttribute(t *testing.T) { err: "not found", additionalSearch: true, }, + { + name: "success search by FilePath with leading slash (with additional search)", + firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), + secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), + reqAttrKey: attrFilePath, + reqAttrValue: "/cat.jpg", + additionalSearch: true, + }, } { t.Run(tc.name, func(t *testing.T) { obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] @@ -422,6 +439,17 @@ func TestNeedSearchByFileName(t *testing.T) { } } +func TestPrepareFileName(t *testing.T) { + fileName := "/cat.jpg" + expected := "cat.jpg" + actual := prepareFileName(fileName) + require.Equal(t, expected, actual) + + fileName = "cat.jpg" + actual = prepareFileName(fileName) + require.Equal(t, expected, actual) +} + func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) From 466f3a9531002cf8d93e8d0690133c4a7f07a639 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Wed, 5 Feb 2025 14:40:31 +0300 Subject: [PATCH 526/548] [#174] Port release v0.32.3 changelog Signed-off-by: Roman Loginov --- CHANGELOG.md | 8 +++++++- VERSION | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51080bb..2025b6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ This document outlines major changes between releases. - Add handling quota limit reached error (#187) - Add slash clipping for FileName attribute (#174) +## [0.32.3] - 2025-02-05 + +### Added +- Add slash clipping for FileName attribute (#174) + ## [0.32.2] - 2025-02-03 ### Fixed @@ -200,4 +205,5 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs [0.32.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...v0.32.0 [0.32.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.0...v0.32.1 [0.32.2]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.1...v0.32.2 -[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.2...master \ No newline at end of file +[0.32.3]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.2...v0.32.3 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.32.3...master \ No newline at end of file diff --git a/VERSION b/VERSION index c6a2605..2c768c5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.32.2 +v0.32.3 From b362793e79880ca1083f34766fd56d08a55c3b13 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 11 Feb 2025 18:43:31 +0300 Subject: [PATCH 527/548] [#195] Use datapath tag in FrostFS pools logs Signed-off-by: Alex Vanin --- cmd/http-gw/settings.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 12d73d6..0a42a90 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -661,8 +661,8 @@ func (a *app) initPools(ctx context.Context) { errorThreshold = defaultPoolErrorThreshold } prm.SetErrorThreshold(errorThreshold) - prm.SetLogger(a.log) - prmTree.SetLogger(a.log) + prm.SetLogger(a.log.With(logs.TagField(logs.TagDatapath))) + prmTree.SetLogger(a.log.With(logs.TagField(logs.TagDatapath))) prmTree.SetMaxRequestAttempts(a.config().GetInt(cfgTreePoolMaxAttempts)) From 8bfaa841243e7d7dc22898dc45825e068b15bb08 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Tue, 18 Feb 2025 12:53:04 +0300 Subject: [PATCH 528/548] [#216] Remove http2 forcing fasthttp doesn't support http2 which causes errors when we enable it Signed-off-by: Nikita Zinkevich --- cmd/http-gw/server.go | 1 - go.mod | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/http-gw/server.go b/cmd/http-gw/server.go index 694e9ee..f8a20d9 100644 --- a/cmd/http-gw/server.go +++ b/cmd/http-gw/server.go @@ -74,7 +74,6 @@ func newServer(ctx context.Context, serverInfo ServerInfo) (*server, error) { ln = tls.NewListener(ln, &tls.Config{ GetCertificate: tlsProvider.GetCertificate, - NextProtos: []string{"h2"}, // required to enable HTTP/2 requests in `http.Serve` }) } diff --git a/go.mod b/go.mod index b3f590e..275ab52 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/net v0.30.0 golang.org/x/sys v0.28.0 google.golang.org/grpc v1.69.2 ) @@ -125,6 +124,7 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.31.0 // indirect + golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect From f9c5dc52604f42c800e7e8d9ec536f60e822dd45 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Tue, 18 Feb 2025 13:24:20 +0300 Subject: [PATCH 529/548] [#216] Rework http2 test to be tls test Signed-off-by: Nikita Zinkevich --- cmd/http-gw/server_test.go | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/cmd/http-gw/server_test.go b/cmd/http-gw/server_test.go index a937366..6f92f17 100644 --- a/cmd/http-gw/server_test.go +++ b/cmd/http-gw/server_test.go @@ -18,7 +18,7 @@ import ( "time" "github.com/stretchr/testify/require" - "golang.org/x/net/http2" + "github.com/valyala/fasthttp" ) const ( @@ -26,14 +26,10 @@ const ( expHeaderValue = "Bar" ) -func TestHTTP2TLS(t *testing.T) { +func TestHTTP_TLS(t *testing.T) { ctx := context.Background() certPath, keyPath := prepareTestCerts(t) - srv := &http.Server{ - Handler: http.HandlerFunc(testHandler), - } - tlsListener, err := newServer(ctx, ServerInfo{ Address: ":0", TLS: ServerTLSInfo{ @@ -47,37 +43,34 @@ func TestHTTP2TLS(t *testing.T) { addr := fmt.Sprintf("https://localhost:%d", port) go func() { - _ = srv.Serve(tlsListener.Listener()) + _ = fasthttp.Serve(tlsListener.Listener(), testHandler) }() - // Server is running, now send HTTP/2 request - tlsClientConfig := &tls.Config{ InsecureSkipVerify: true, } - cliHTTP1 := http.Client{Transport: &http.Transport{TLSClientConfig: tlsClientConfig}} - cliHTTP2 := http.Client{Transport: &http2.Transport{TLSClientConfig: tlsClientConfig}} + cliHTTP := http.Client{Transport: &http.Transport{}} + cliHTTPS := http.Client{Transport: &http.Transport{TLSClientConfig: tlsClientConfig}} req, err := http.NewRequest("GET", addr, nil) require.NoError(t, err) req.Header[expHeaderKey] = []string{expHeaderValue} - resp, err := cliHTTP1.Do(req) + resp, err := cliHTTPS.Do(req) require.NoError(t, err) require.Equal(t, http.StatusOK, resp.StatusCode) - resp, err = cliHTTP2.Do(req) - require.NoError(t, err) - require.Equal(t, http.StatusOK, resp.StatusCode) + _, err = cliHTTP.Do(req) + require.ErrorContains(t, err, "failed to verify certificate") } -func testHandler(resp http.ResponseWriter, req *http.Request) { - hdr, ok := req.Header[expHeaderKey] - if !ok || len(hdr) != 1 || hdr[0] != expHeaderValue { - resp.WriteHeader(http.StatusBadRequest) +func testHandler(ctx *fasthttp.RequestCtx) { + hdr := ctx.Request.Header.Peek(expHeaderKey) + if len(hdr) == 0 || string(hdr) != expHeaderValue { + ctx.Response.SetStatusCode(http.StatusBadRequest) } else { - resp.WriteHeader(http.StatusOK) + ctx.Response.SetStatusCode(http.StatusOK) } } From a651b5823f7f3b445315c680ad54942cac0c4b0e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 21 Feb 2025 16:11:01 +0300 Subject: [PATCH 530/548] [#219] Use zaptest.Logger Use zaptest to get logs which get printed only if a test fails or if you ran go test -v. Dont use zaptest.Logger for fuzz otherwise ngfuzz/libfuzz crashes Signed-off-by: Denis Kirillov --- internal/handler/handler_fuzz_test.go | 3 ++- internal/handler/handler_test.go | 21 ++++++++++----------- internal/handler/multipart_test.go | 14 ++------------ 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/internal/handler/handler_fuzz_test.go b/internal/handler/handler_fuzz_test.go index d71e8b0..ff38b11 100644 --- a/internal/handler/handler_fuzz_test.go +++ b/internal/handler/handler_fuzz_test.go @@ -21,6 +21,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" go_fuzz_utils "github.com/trailofbits/go-fuzz-utils" "github.com/valyala/fasthttp" + "go.uber.org/zap" ) const ( @@ -125,7 +126,7 @@ func maybeFillRandom(tp *go_fuzz_utils.TypeProvider, initValue string) (string, } func upload(tp *go_fuzz_utils.TypeProvider) (context.Context, *handlerContext, cid.ID, *fasthttp.RequestCtx, string, string, string, error) { - hc, err := prepareHandlerContext() + hc, err := prepareHandlerContextBase(zap.NewExample()) if err != nil { return nil, nil, cid.ID{}, nil, "", "", "", err } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 1638f9f..383dcd9 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/require" "github.com/valyala/fasthttp" "go.uber.org/zap" + "go.uber.org/zap/zaptest" ) type treeServiceMock struct { @@ -112,12 +113,13 @@ func (hc *handlerContext) Handler() *Handler { return hc.h } -func prepareHandlerContext() (*handlerContext, error) { - logger, err := zap.NewDevelopment() - if err != nil { - return nil, err - } +func prepareHandlerContext(t *testing.T) *handlerContext { + hc, err := prepareHandlerContextBase(zaptest.NewLogger(t)) + require.NoError(t, err) + return hc +} +func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { key, err := keys.NewPrivateKey() if err != nil { return nil, err @@ -196,8 +198,7 @@ func (hc *handlerContext) prepareContainer(name string, basicACL acl.Basic) (cid } func TestBasic(t *testing.T) { - hc, err := prepareHandlerContext() - require.NoError(t, err) + hc := prepareHandlerContext(t) bktName := "bucket" cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended) @@ -279,8 +280,7 @@ func TestBasic(t *testing.T) { } func TestFindObjectByAttribute(t *testing.T) { - hc, err := prepareHandlerContext() - require.NoError(t, err) + hc := prepareHandlerContext(t) hc.cfg.additionalSearch = true bktName := "bucket" @@ -377,8 +377,7 @@ func TestFindObjectByAttribute(t *testing.T) { } func TestNeedSearchByFileName(t *testing.T) { - hc, err := prepareHandlerContext() - require.NoError(t, err) + hc := prepareHandlerContext(t) for _, tc := range []struct { name string diff --git a/internal/handler/multipart_test.go b/internal/handler/multipart_test.go index 431d0d6..d7f52f4 100644 --- a/internal/handler/multipart_test.go +++ b/internal/handler/multipart_test.go @@ -60,12 +60,7 @@ func BenchmarkAll(b *testing.B) { func defaultMultipart(filename string) error { r, bound := multipartFile(filename) - logger, err := zap.NewProduction() - if err != nil { - return err - } - - file, err := fetchMultipartFileDefault(logger, r, bound) + file, err := fetchMultipartFileDefault(zap.NewNop(), r, bound) if err != nil { return err } @@ -87,12 +82,7 @@ func TestName(t *testing.T) { func customMultipart(filename string) error { r, bound := multipartFile(filename) - logger, err := zap.NewProduction() - if err != nil { - return err - } - - file, err := fetchMultipartFile(logger, r, bound) + file, err := fetchMultipartFile(zap.NewNop(), r, bound) if err != nil { return err } From cc6055bd27ec7f71d3123f4d61874eb1e41ac336 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Wed, 12 Feb 2025 11:08:59 +0300 Subject: [PATCH 531/548] [#211] Add IO tags Signed-off-by: Marina Biryukova --- cmd/http-gw/settings.go | 3 +++ go.mod | 1 + go.sum | 6 ++++-- internal/service/frostfs/frostfs.go | 13 ++++++++----- internal/service/frostfs/tree_pool_wrapper.go | 5 +++-- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 0a42a90..69ecce2 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -21,6 +21,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "github.com/spf13/pflag" @@ -670,6 +671,8 @@ func (a *app) initPools(ctx context.Context) { grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), grpc.WithContextDialer(a.settings.dialerSource.GrpcContextDialer()), + grpc.WithChainUnaryInterceptor(qostagging.NewUnaryClientInteceptor()), + grpc.WithChainStreamInterceptor(qostagging.NewStreamClientInterceptor()), } prm.SetGRPCDialOptions(interceptors...) prmTree.SetGRPCDialOptions(interceptors...) diff --git a/go.mod b/go.mod index 275ab52..0ace5f2 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 + git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 diff --git a/go.sum b/go.sum index 9ee2d9c..a2121ab 100644 --- a/go.sum +++ b/go.sum @@ -42,10 +42,12 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 h1:/Z8DfbLZXp7exUQWUKoG/9tbFdI9d5lV1qSReaYoG8I= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe h1:81gDNdWNLP24oMQukRiCE9R1wGSh0l0dRq3F1W+Oesc= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index 4cf45a4..9115930 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -20,6 +21,8 @@ import ( "google.golang.org/grpc/status" ) +const clientIOTag = "client" + // FrostFS represents virtual connection to the FrostFS network. // It is used to provide an interface to dependent packages // which work with FrostFS. @@ -67,7 +70,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) prmPut.UseBearer(*prm.BearerToken) } - idObj, err := x.pool.PutObject(ctx, prmPut) + idObj, err := x.pool.PutObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmPut) if err != nil { return oid.ID{}, handleObjectError("save object via connection pool", err) } @@ -100,7 +103,7 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o prmHead.UseBearer(*prm.BearerToken) } - res, err := x.pool.HeadObject(ctx, prmHead) + res, err := x.pool.HeadObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmHead) if err != nil { return nil, handleObjectError("read object header via connection pool", err) } @@ -120,7 +123,7 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han prmGet.UseBearer(*prm.BearerToken) } - res, err := x.pool.GetObject(ctx, prmGet) + res, err := x.pool.GetObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmGet) if err != nil { return nil, handleObjectError("init full object reading via connection pool", err) } @@ -145,7 +148,7 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) ( prmRange.UseBearer(*prm.BearerToken) } - res, err := x.pool.ObjectRange(ctx, prmRange) + res, err := x.pool.ObjectRange(qostagging.ContextWithIOTag(ctx, clientIOTag), prmRange) if err != nil { return nil, handleObjectError("init payload range reading via connection pool", err) } @@ -166,7 +169,7 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch prmSearch.UseBearer(*prm.BearerToken) } - res, err := x.pool.SearchObjects(ctx, prmSearch) + res, err := x.pool.SearchObjects(qostagging.ContextWithIOTag(ctx, clientIOTag), prmSearch) if err != nil { return nil, handleObjectError("init object search via connection pool", err) } diff --git a/internal/service/frostfs/tree_pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go index 410acda..89afc3c 100644 --- a/internal/service/frostfs/tree_pool_wrapper.go +++ b/internal/service/frostfs/tree_pool_wrapper.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" apitree "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/tree" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" ) @@ -61,7 +62,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([ BearerToken: getBearer(ctx), } - nodes, err := w.p.GetNodes(ctx, poolPrm) + nodes, err := w.p.GetNodes(qostagging.ContextWithIOTag(ctx, clientIOTag), poolPrm) if err != nil { return nil, handleError(err) } @@ -120,7 +121,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, poolPrm.RootID = nil } - subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) + subTreeReader, err := w.p.GetSubTree(qostagging.ContextWithIOTag(ctx, clientIOTag), poolPrm) if err != nil { return nil, handleError(err) } From 9cf2a4f0e0011bf0ae87482a36b2055405c736e8 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Sun, 9 Feb 2025 21:48:32 +0300 Subject: [PATCH 532/548] [#197] Add a leading slash to the FilePath attribute According to the frostfs api specification, the File Path attribute must start with a leading slash. More info: https://git.frostfs.info/TrueCloudLab/frostfs-api Signed-off-by: Roman Loginov --- internal/handler/handler.go | 27 ++++++++++++++++++++++---- internal/handler/handler_test.go | 33 +++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 532cdc4..179cf60 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -253,9 +253,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte return } - if key == attrFileName { - val = prepareFileName(val) - } + val = prepareAtribute(key, val) log = log.With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val)) @@ -298,7 +296,8 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cn switch { case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) - return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, prepareFileName(attrVal)) + attrVal = prepareAtribute(attrFileName, attrVal) + return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal) case errors.Is(err, io.EOF): log.Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("object not found: %w", err) @@ -319,6 +318,18 @@ func (h *Handler) needSearchByFileName(key, val string) bool { return strings.HasPrefix(val, "/") && strings.Count(val, "/") == 1 || !strings.Contains(val, "/") } +func prepareAtribute(attrKey, attrVal string) string { + if attrKey == attrFileName { + return prepareFileName(attrVal) + } + + if attrKey == attrFilePath { + return prepareFilePath(attrVal) + } + + return attrVal +} + func prepareFileName(fileName string) string { if strings.HasPrefix(fileName, "/") { return fileName[1:] @@ -327,6 +338,14 @@ func prepareFileName(fileName string) string { return fileName } +func prepareFilePath(filePath string) string { + if !strings.HasPrefix(filePath, "/") { + return "/" + filePath + } + + return filePath +} + // resolveContainer decode container id, if it's not a valid container id // then trey to resolve name using provided resolver. func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 383dcd9..ab2cd9f 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -220,8 +220,10 @@ func TestBasic(t *testing.T) { require.NoError(t, err) obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] - attr := prepareObjectAttributes(object.AttributeFilePath, objFileName) - obj.SetAttributes(append(obj.Attributes(), attr)...) + fileName := prepareObjectAttributes(object.AttributeFileName, objFileName) + filePath := prepareObjectAttributes(object.AttributeFilePath, objFilePath) + obj.SetAttributes(append(obj.Attributes(), fileName)...) + obj.SetAttributes(append(obj.Attributes(), filePath)...) t.Run("get", func(t *testing.T) { r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID) @@ -241,7 +243,11 @@ func TestBasic(t *testing.T) { hc.Handler().DownloadByAttribute(r) require.Equal(t, content, string(r.Response.Body())) - r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, "/"+objFileName) + r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath) + hc.Handler().DownloadByAttribute(r) + require.Equal(t, content, string(r.Response.Body())) + + r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName) hc.Handler().DownloadByAttribute(r) require.Equal(t, content, string(r.Response.Body())) }) @@ -252,7 +258,12 @@ func TestBasic(t *testing.T) { require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) - r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, "/"+objFileName) + r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath) + hc.Handler().HeadByAttribute(r) + require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) + require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + + r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName) hc.Handler().HeadByAttribute(r) require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) @@ -266,7 +277,7 @@ func TestBasic(t *testing.T) { zipReader, err := zip.NewReader(readerAt, int64(len(r.Response.Body()))) require.NoError(t, err) require.Len(t, zipReader.File, 1) - require.Equal(t, objFileName, zipReader.File[0].Name) + require.Equal(t, objFilePath, zipReader.File[0].Name) f, err := zipReader.File[0].Open() require.NoError(t, err) defer func() { @@ -449,6 +460,17 @@ func TestPrepareFileName(t *testing.T) { require.Equal(t, expected, actual) } +func TestPrepareFilePath(t *testing.T) { + filePath := "cat.jpg" + expected := "/cat.jpg" + actual := prepareFilePath(filePath) + require.Equal(t, expected, actual) + + filePath = "/cat.jpg" + actual = prepareFilePath(filePath) + require.Equal(t, expected, actual) +} + func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) @@ -492,6 +514,7 @@ const ( keyAttr = "User-Attribute" valAttr = "user value" objFileName = "newFile.txt" + objFilePath = "/newFile.txt" ) func fillMultipartBody(r *fasthttp.RequestCtx, content string) error { From 9ef6b06e91899aa4f2f3c32dd0179f3cf9282c2f Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Thu, 27 Feb 2025 12:13:12 +0300 Subject: [PATCH 533/548] [#212] Support CORS container for CORS settings Signed-off-by: Marina Biryukova --- cmd/http-gw/app.go | 206 ++++---- cmd/http-gw/integration_test.go | 21 +- cmd/http-gw/settings.go | 43 +- config/config.env | 6 + config/config.yaml | 7 + docs/gate-configuration.md | 20 +- internal/cache/cors.go | 62 +++ internal/data/cors.go | 18 + internal/handler/cors.go | 342 ++++++++++++++ internal/handler/cors_test.go | 440 ++++++++++++++++++ internal/handler/frostfs_mock.go | 4 + internal/handler/handler.go | 29 +- internal/handler/handler_test.go | 60 ++- internal/logs/logs.go | 68 +-- internal/service/frostfs/frostfs.go | 13 +- internal/service/frostfs/tree_pool_wrapper.go | 5 +- resolver/resolver.go | 52 +-- utils/attributes.go | 11 + 18 files changed, 1204 insertions(+), 203 deletions(-) create mode 100644 internal/cache/cors.go create mode 100644 internal/data/cors.go create mode 100644 internal/handler/cors.go create mode 100644 internal/handler/cors_test.go diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 103c72b..c75f9d8 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -17,6 +17,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" @@ -30,6 +31,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -65,6 +67,8 @@ type ( settings *appSettings loggerSettings *loggerSettings bucketCache *cache.BucketCache + handle *handler.Handler + corsCnrID cid.ID servers []Server unbindServers []ServerInfo @@ -105,12 +109,7 @@ type ( bufferMaxSizeForPut uint64 namespaceHeader string defaultNamespaces []string - corsAllowOrigin string - corsAllowMethods []string - corsAllowHeaders []string - corsExposeHeaders []string - corsAllowCredentials bool - corsMaxAge int + cors *data.CORSRule enableFilepathFallback bool } @@ -122,15 +121,6 @@ type ( logLevel zap.AtomicLevel tagsConfig *tagsConfig } - - CORS struct { - AllowOrigin string - AllowMethods []string - AllowHeaders []string - ExposeHeaders []string - AllowCredentials bool - MaxAge int - } ) func newLogLevel(v *viper.Viper) zap.AtomicLevel { @@ -251,6 +241,7 @@ func newApp(ctx context.Context, cfg *appCfg) App { a.initResolver() a.initMetrics() a.initTracing(ctx) + a.initContainers(ctx) return a } @@ -259,6 +250,14 @@ func (a *app) config() *viper.Viper { return a.cfg.config() } +func (a *app) initContainers(ctx context.Context) { + corsCnrID, err := a.fetchContainerID(ctx, cfgContainersCORS) + if err != nil { + a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err), logs.TagField(logs.TagApp)) + } + a.corsCnrID = *corsCnrID +} + func (a *app) initAppSettings(lc *logLevelConfig) { a.settings = &appSettings{ reconnectInterval: fetchReconnectInterval(a.config()), @@ -278,12 +277,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { namespaceHeader := v.GetString(cfgResolveNamespaceHeader) defaultNamespaces := fetchDefaultNamespaces(v) indexPage, indexEnabled := fetchIndexPageTemplate(v, l) - corsAllowOrigin := v.GetString(cfgCORSAllowOrigin) - corsAllowMethods := v.GetStringSlice(cfgCORSAllowMethods) - corsAllowHeaders := v.GetStringSlice(cfgCORSAllowHeaders) - corsExposeHeaders := v.GetStringSlice(cfgCORSExposeHeaders) - corsAllowCredentials := v.GetBool(cfgCORSAllowCredentials) - corsMaxAge := fetchCORSMaxAge(v) + cors := fetchCORSConfig(v) enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback) s.mu.Lock() @@ -298,12 +292,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { s.defaultNamespaces = defaultNamespaces s.returnIndexPage = indexEnabled s.indexPageTemplate = indexPage - s.corsAllowOrigin = corsAllowOrigin - s.corsAllowMethods = corsAllowMethods - s.corsAllowHeaders = corsAllowHeaders - s.corsExposeHeaders = corsExposeHeaders - s.corsAllowCredentials = corsAllowCredentials - s.corsMaxAge = corsMaxAge + s.cors = cors s.enableFilepathFallback = enableFilepathFallback } @@ -350,26 +339,33 @@ func (s *appSettings) IndexPageTemplate() string { return s.indexPageTemplate } -func (s *appSettings) CORS() CORS { +func (s *appSettings) CORS() *data.CORSRule { s.mu.RLock() defer s.mu.RUnlock() - allowMethods := make([]string, len(s.corsAllowMethods)) - copy(allowMethods, s.corsAllowMethods) + if s.cors == nil { + return nil + } - allowHeaders := make([]string, len(s.corsAllowHeaders)) - copy(allowHeaders, s.corsAllowHeaders) + allowMethods := make([]string, len(s.cors.AllowedMethods)) + copy(allowMethods, s.cors.AllowedMethods) - exposeHeaders := make([]string, len(s.corsExposeHeaders)) - copy(exposeHeaders, s.corsExposeHeaders) + allowHeaders := make([]string, len(s.cors.AllowedHeaders)) + copy(allowHeaders, s.cors.AllowedHeaders) - return CORS{ - AllowOrigin: s.corsAllowOrigin, - AllowMethods: allowMethods, - AllowHeaders: allowHeaders, - ExposeHeaders: exposeHeaders, - AllowCredentials: s.corsAllowCredentials, - MaxAge: s.corsMaxAge, + exposeHeaders := make([]string, len(s.cors.ExposeHeaders)) + copy(exposeHeaders, s.cors.ExposeHeaders) + + allowOrigins := make([]string, len(s.cors.AllowedOrigins)) + copy(allowOrigins, s.cors.AllowedOrigins) + + return &data.CORSRule{ + AllowedOrigins: allowOrigins, + AllowedMethods: allowMethods, + AllowedHeaders: allowHeaders, + ExposeHeaders: exposeHeaders, + AllowedCredentials: s.cors.AllowedCredentials, + MaxAgeSeconds: s.cors.MaxAgeSeconds, } } @@ -391,15 +387,15 @@ func (s *appSettings) NamespaceHeader() string { return s.namespaceHeader } -func (s *appSettings) FormContainerZone(ns string) (zone string, isDefault bool) { +func (s *appSettings) FormContainerZone(ns string) string { s.mu.RLock() namespaces := s.defaultNamespaces s.mu.RUnlock() if slices.Contains(namespaces, ns) { - return v2container.SysAttributeZoneDefault, true + return v2container.SysAttributeZoneDefault } - return ns + ".ns", false + return ns + ".ns" } func (s *appSettings) EnableFilepathFallback() bool { @@ -420,7 +416,6 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) { resolveCfg := &resolver.Config{ FrostFS: frostfs.NewResolverFrostFS(a.pool), RPCAddress: a.config().GetString(cfgRPCEndpoint), - Settings: a.settings, } order := a.config().GetStringSlice(cfgResolveOrder) @@ -606,10 +601,8 @@ func (a *app) Serve() { close(a.webDone) }() - handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) - // Configure router. - a.configureRouter(handle) + a.configureRouter(workerPool) a.startServices() a.initServers(a.ctx) @@ -730,7 +723,9 @@ func (a *app) stopServices() { } } -func (a *app) configureRouter(h *handler.Handler) { +func (a *app) configureRouter(workerPool *ants.Pool) { + a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) + r := router.New() r.RedirectTrailingSlash = true r.NotFound = func(r *fasthttp.RequestCtx) { @@ -740,21 +735,21 @@ func (a *app) configureRouter(h *handler.Handler) { handler.ResponseError(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.addMiddlewares(h.Upload)) - r.OPTIONS("/upload/{cid}", a.addPreflight()) + r.POST("/upload/{cid}", a.addMiddlewares(a.handle.Upload)) + r.OPTIONS("/upload/{cid}", a.addPreflight(a.handle.Preflight)) a.log.Info(logs.AddedPathUploadCid, logs.TagField(logs.TagApp)) - r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(h.DownloadByAddressOrBucketName)) - r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(h.HeadByAddressOrBucketName)) - r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight()) + r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(a.handle.DownloadByAddressOrBucketName)) + r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(a.handle.HeadByAddressOrBucketName)) + r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight(a.handle.Preflight)) a.log.Info(logs.AddedPathGetCidOid, logs.TagField(logs.TagApp)) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.DownloadByAttribute)) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.HeadByAttribute)) - r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(a.handle.DownloadByAttribute)) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(a.handle.HeadByAttribute)) + r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight(a.handle.Preflight)) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal, logs.TagField(logs.TagApp)) - r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZip)) - r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) - r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadTar)) - r.OPTIONS("/tar/{cid}/{prefix:*}", a.addPreflight()) + r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(a.handle.DownloadZip)) + r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight(a.handle.Preflight)) + r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(a.handle.DownloadTar)) + r.OPTIONS("/tar/{cid}/{prefix:*}", a.addPreflight(a.handle.Preflight)) a.log.Info(logs.AddedPathZipCidPrefix, logs.TagField(logs.TagApp)) a.webServer.Handler = r.Handler @@ -777,14 +772,14 @@ func (a *app) addMiddlewares(h fasthttp.RequestHandler) fasthttp.RequestHandler return h } -func (a *app) addPreflight() fasthttp.RequestHandler { +func (a *app) addPreflight(h fasthttp.RequestHandler) fasthttp.RequestHandler { list := []func(fasthttp.RequestHandler) fasthttp.RequestHandler{ a.tracer, a.logger, + a.canonicalizer, a.reqNamespace, } - h := a.preflightHandler for i := len(list) - 1; i >= 0; i-- { h = list[i](h) } @@ -792,46 +787,16 @@ func (a *app) addPreflight() fasthttp.RequestHandler { return h } -func (a *app) preflightHandler(c *fasthttp.RequestCtx) { - cors := a.settings.CORS() - setCORSHeaders(c, cors) -} - func (a *app) cors(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(c *fasthttp.RequestCtx) { h(c) code := c.Response.StatusCode() if code >= fasthttp.StatusOK && code < fasthttp.StatusMultipleChoices { - cors := a.settings.CORS() - setCORSHeaders(c, cors) + a.handle.SetCORSHeaders(c) } } } -func setCORSHeaders(c *fasthttp.RequestCtx, cors CORS) { - c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(cors.MaxAge)) - - if len(cors.AllowOrigin) != 0 { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, cors.AllowOrigin) - } - - if len(cors.AllowMethods) != 0 { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(cors.AllowMethods, ",")) - } - - if len(cors.AllowHeaders) != 0 { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, strings.Join(cors.AllowHeaders, ",")) - } - - if len(cors.ExposeHeaders) != 0 { - c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(cors.ExposeHeaders, ",")) - } - - if cors.AllowCredentials { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") - } -} - func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { return func(req *fasthttp.RequestCtx) { requiredFields := []zap.Field{zap.Uint64("id", req.ID())} @@ -930,11 +895,13 @@ func (a *app) reqNamespace(h fasthttp.RequestHandler) fasthttp.RequestHandler { func (a *app) AppParams() *handler.AppParams { return &handler.AppParams{ - Logger: a.log, - FrostFS: frostfs.NewFrostFS(a.pool), - Owner: a.owner, - Resolver: a.resolver, - Cache: a.bucketCache, + Logger: a.log, + FrostFS: frostfs.NewFrostFS(a.pool), + Owner: a.owner, + Resolver: a.resolver, + Cache: a.bucketCache, + CORSCnrID: a.corsCnrID, + CORSCache: cache.NewCORSCache(getCORSCacheOptions(a.config(), a.log)), } } @@ -1135,3 +1102,44 @@ func (a *app) tryReconnect(ctx context.Context, sr *fasthttp.Server) bool { return len(a.unbindServers) == 0 } + +func (a *app) fetchContainerID(ctx context.Context, cfgKey string) (id *cid.ID, err error) { + cnrID, err := a.resolveContainerID(ctx, cfgKey) + if err != nil { + return nil, err + } + + err = checkContainerExists(ctx, *cnrID, a.pool) + if err != nil { + return nil, err + } + + return cnrID, nil +} + +func (a *app) resolveContainerID(ctx context.Context, cfgKey string) (*cid.ID, error) { + containerString := a.config().GetString(cfgKey) + + id := new(cid.ID) + if err := id.DecodeString(containerString); err != nil { + i := strings.Index(containerString, ".") + if i < 0 { + return nil, fmt.Errorf("invalid container address: %s", containerString) + } + + if id, err = a.resolver.Resolve(ctx, containerString[i+1:], containerString[:i]); err != nil { + return nil, fmt.Errorf("resolve container address %s: %w", containerString, err) + } + } + + return id, nil +} + +func checkContainerExists(ctx context.Context, id cid.ID, frostFSPool *pool.Pool) error { + prm := pool.PrmContainerGet{ + ContainerID: id, + } + + _, err := frostFSPool.GetContainer(ctx, prm) + return err +} diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 2596bee..20b4c8b 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -43,9 +43,10 @@ type putResponse struct { } const ( - testContainerName = "friendly" - testListenAddress = "localhost:8082" - testHost = "http://" + testListenAddress + testContainerName = "friendly" + testListenAddress = "localhost:8082" + testHost = "http://" + testListenAddress + testCORSContainerName = "cors" ) func TestIntegration(t *testing.T) { @@ -76,10 +77,14 @@ func TestIntegration(t *testing.T) { registerUser(t, ctx, aioContainer, file.Name()) } + // Creating CORS container + clientPool := getPool(ctx, t, key) + _, err = createContainer(ctx, t, clientPool, ownerID, testCORSContainerName) + require.NoError(t, err, version) + // See the logs from the command execution. server, cancel := runServer(file.Name()) - clientPool := getPool(ctx, t, key) - CID, err := createContainer(ctx, t, clientPool, ownerID) + CID, err := createContainer(ctx, t, clientPool, ownerID, testContainerName) require.NoError(t, err, version) jsonToken, binaryToken := makeBearerTokens(t, key, ownerID, version) @@ -110,6 +115,8 @@ func runServer(pathToWallet string) (App, context.CancelFunc) { v.config().Set(cfgWalletPath, pathToWallet) v.config().Set(cfgWalletPassphrase, "") + v.config().Set(cfgContainersCORS, testCORSContainerName+"."+containerv2.SysAttributeZoneDefault) + application := newApp(cancelCtx, v) go application.Serve() @@ -477,7 +484,7 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool return clientPool } -func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID) (cid.ID, error) { +func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, name string) (cid.ID, error) { var policy netmap.PlacementPolicy err := policy.DecodeString("REP 1") require.NoError(t, err) @@ -491,7 +498,7 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o container.SetCreationTime(&cnr, time.Now()) var domain container.Domain - domain.SetName(testContainerName) + domain.SetName(name) cnr.SetAttribute(containerv2.SysAttributeName, domain.Name()) cnr.SetAttribute(containerv2.SysAttributeZone, domain.Zone()) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 69ecce2..132c627 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -16,6 +16,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" @@ -155,18 +156,21 @@ const ( cfgBucketsCacheLifetime = "cache.buckets.lifetime" cfgBucketsCacheSize = "cache.buckets.size" cfgNetmapCacheLifetime = "cache.netmap.lifetime" + cfgCORSCacheLifetime = "cache.cors.lifetime" + cfgCORSCacheSize = "cache.cors.size" // Bucket resolving options. cfgResolveNamespaceHeader = "resolve_bucket.namespace_header" cfgResolveDefaultNamespaces = "resolve_bucket.default_namespaces" // CORS. - cfgCORSAllowOrigin = "cors.allow_origin" - cfgCORSAllowMethods = "cors.allow_methods" - cfgCORSAllowHeaders = "cors.allow_headers" - cfgCORSExposeHeaders = "cors.expose_headers" - cfgCORSAllowCredentials = "cors.allow_credentials" - cfgCORSMaxAge = "cors.max_age" + cfgCORS = "cors" + cfgCORSAllowOrigin = cfgCORS + ".allow_origin" + cfgCORSAllowMethods = cfgCORS + ".allow_methods" + cfgCORSAllowHeaders = cfgCORS + ".allow_headers" + cfgCORSExposeHeaders = cfgCORS + ".expose_headers" + cfgCORSAllowCredentials = cfgCORS + ".allow_credentials" + cfgCORSMaxAge = cfgCORS + ".max_age" // Multinet. cfgMultinetEnabled = "multinet.enabled" @@ -179,6 +183,9 @@ const ( cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support" + // Containers. + cfgContainersCORS = "containers.cors" + // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -759,6 +766,15 @@ func getNetmapCacheOptions(v *viper.Viper, l *zap.Logger) *cache.NetmapCacheConf return cacheCfg } +func getCORSCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { + cacheCfg := cache.DefaultCORSConfig(l) + + cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgCORSCacheLifetime, cacheCfg.Lifetime) + cacheCfg.Size = fetchCacheSize(v, l, cfgCORSCacheSize, cacheCfg.Size) + + return cacheCfg +} + func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration { if v.IsSet(cfgEntry) { lifetime := v.GetDuration(cfgEntry) @@ -854,3 +870,18 @@ func fetchArchiveCompression(v *viper.Viper) bool { } return v.GetBool(cfgArchiveCompression) } + +func fetchCORSConfig(v *viper.Viper) *data.CORSRule { + if !v.IsSet(cfgCORS) { + return nil + } + + return &data.CORSRule{ + AllowedOrigins: []string{v.GetString(cfgCORSAllowOrigin)}, + AllowedMethods: v.GetStringSlice(cfgCORSAllowMethods), + AllowedHeaders: v.GetStringSlice(cfgCORSAllowHeaders), + ExposeHeaders: v.GetStringSlice(cfgCORSExposeHeaders), + AllowedCredentials: v.GetBool(cfgCORSAllowCredentials), + MaxAgeSeconds: fetchCORSMaxAge(v), + } +} diff --git a/config/config.env b/config/config.env index af0eba1..0ff2dec 100644 --- a/config/config.env +++ b/config/config.env @@ -129,6 +129,9 @@ HTTP_GW_CACHE_BUCKETS_LIFETIME=1m HTTP_GW_CACHE_BUCKETS_SIZE=1000 # Cache which stores netmap HTTP_GW_CACHE_NETMAP_LIFETIME=1m +# Cache which stores container CORS configurations +HTTP_GW_CACHE_CORS_LIFETIME=5m +HTTP_GW_CACHE_CORS_SIZE=1000 # Header to determine zone to resolve bucket name HTTP_GW_RESOLVE_BUCKET_NAMESPACE_HEADER=X-Frostfs-Namespace @@ -172,3 +175,6 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true + +# Containers properties +HTTP_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj diff --git a/config/config.yaml b/config/config.yaml index 8c51591..05bba2e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -156,6 +156,10 @@ cache: # Cache which stores netmap netmap: lifetime: 1m + # Cache which stores container CORS configurations + cors: + lifetime: 5m + size: 1000 resolve_bucket: namespace_header: X-Frostfs-Namespace @@ -191,3 +195,6 @@ features: enable_filepath_fallback: false # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service tree_pool_netmap_support: true + +containers: + cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 191e9bb..628d3c7 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -60,6 +60,7 @@ $ cat http.log | `index_page` | [Index page configuration](#index_page-section) | | `multinet` | [Multinet configuration](#multinet-section) | | `features` | [Features configuration](#features-section) | +| `containers` | [Containers configuration](#containers-section) | # General section @@ -382,12 +383,16 @@ cache: size: 1000 netmap: lifetime: 1m + cors: + lifetime: 5m + size: 1000 ``` | Parameter | Type | Default value | Description | |-----------|-----------------------------------|---------------------------------|---------------------------------------------------------------------------| | `buckets` | [Cache config](#cache-subsection) | `lifetime: 60s`
`size: 1000` | Cache which contains mapping of bucket name to bucket info. | | `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. | +| `cors` | [Cache config](#cache-subsection) | `lifetime: 5m`
`size: 1000` | Cache which stores container CORS configurations. | #### `cache` subsection @@ -441,7 +446,7 @@ index_page: # `cors` section Parameters for CORS (used in OPTIONS requests and responses in all handlers). -If values are not set, headers will not be included to response. +If values are not set, settings from CORS container will be used. ```yaml cors: @@ -515,3 +520,16 @@ features: |-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. | | `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | + +# `containers` section + +Section for well-known containers to store data and settings. + +```yaml +containers: + cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-------------|----------|---------------|---------------|-----------------------------------------| +| `cors` | `string` | no | | Container name for CORS configurations. | diff --git a/internal/cache/cors.go b/internal/cache/cors.go new file mode 100644 index 0000000..24465b8 --- /dev/null +++ b/internal/cache/cors.go @@ -0,0 +1,62 @@ +package cache + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/bluele/gcache" + "go.uber.org/zap" +) + +// CORSCache contains cache with CORS objects. +type CORSCache struct { + cache gcache.Cache + logger *zap.Logger +} + +const ( + // DefaultCORSCacheSize is a default maximum number of entries in cache. + DefaultCORSCacheSize = 1e3 + // DefaultCORSCacheLifetime is a default lifetime of entries in cache. + DefaultCORSCacheLifetime = 5 * time.Minute +) + +// DefaultCORSConfig returns new default cache expiration values. +func DefaultCORSConfig(logger *zap.Logger) *Config { + return &Config{ + Size: DefaultCORSCacheSize, + Lifetime: DefaultCORSCacheLifetime, + Logger: logger, + } +} + +// NewCORSCache creates an object of CORSCache. +func NewCORSCache(config *Config) *CORSCache { + gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + return &CORSCache{cache: gc, logger: config.Logger} +} + +// Get returns a cached object. +func (o *CORSCache) Get(cnrID cid.ID) *data.CORSConfiguration { + entry, err := o.cache.Get(cnrID) + if err != nil { + return nil + } + + result, ok := entry.(*data.CORSConfiguration) + if !ok { + o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath)) + return nil + } + + return result +} + +// Put puts an object to cache. +func (o *CORSCache) Put(cnrID cid.ID, cors *data.CORSConfiguration) error { + return o.cache.Set(cnrID, cors) +} diff --git a/internal/data/cors.go b/internal/data/cors.go new file mode 100644 index 0000000..d1b1106 --- /dev/null +++ b/internal/data/cors.go @@ -0,0 +1,18 @@ +package data + +type ( + // CORSConfiguration stores CORS configuration of a request. + CORSConfiguration struct { + CORSRules []CORSRule `xml:"CORSRule" json:"CORSRules"` + } + + // CORSRule stores rules for CORS configuration. + CORSRule struct { + AllowedHeaders []string `xml:"AllowedHeader" json:"AllowedHeaders"` + AllowedMethods []string `xml:"AllowedMethod" json:"AllowedMethods"` + AllowedOrigins []string `xml:"AllowedOrigin" json:"AllowedOrigins"` + ExposeHeaders []string `xml:"ExposeHeader" json:"ExposeHeaders"` + MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty" json:"MaxAgeSeconds,omitempty"` + AllowedCredentials bool `xml:"AllowedCredentials,omitempty" json:"AllowedCredentials,omitempty"` + } +) diff --git a/internal/handler/cors.go b/internal/handler/cors.go new file mode 100644 index 0000000..234ef2a --- /dev/null +++ b/internal/handler/cors.go @@ -0,0 +1,342 @@ +package handler + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "sort" + "strconv" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +const ( + internalIOTag = "internal" + corsFilePathTemplate = "/%s.cors" + wildcard = "*" +) + +var errNoCORS = errors.New("no CORS objects found") + +func (h *Handler) Preflight(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.Preflight") + defer span.End() + + ctx = qostagging.ContextWithIOTag(ctx, internalIOTag) + cidParam, _ := c.UserValue("cid").(string) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("cid", cidParam)) + + origin := c.Request.Header.Peek(fasthttp.HeaderOrigin) + if len(origin) == 0 { + log.Error(logs.EmptyOriginRequestHeader, logs.TagField(logs.TagDatapath)) + ResponseError(c, "Origin request header needed", fasthttp.StatusBadRequest) + return + } + + method := c.Request.Header.Peek(fasthttp.HeaderAccessControlRequestMethod) + if len(method) == 0 { + log.Error(logs.EmptyAccessControlRequestMethodHeader, logs.TagField(logs.TagDatapath)) + ResponseError(c, "Access-Control-Request-Method request header needed", fasthttp.StatusBadRequest) + return + } + + corsRule := h.config.CORS() + if corsRule != nil { + setCORSHeadersFromRule(c, corsRule) + return + } + + corsConfig, err := h.getCORSConfig(ctx, log, cidParam) + if err != nil { + log.Error(logs.CouldNotGetCORSConfiguration, zap.Error(err), logs.TagField(logs.TagDatapath)) + status := fasthttp.StatusInternalServerError + if errors.Is(err, errNoCORS) { + status = fasthttp.StatusNotFound + } + ResponseError(c, "could not get CORS configuration: "+err.Error(), status) + return + } + + var headers []string + requestHeaders := c.Request.Header.Peek(fasthttp.HeaderAccessControlRequestHeaders) + if len(requestHeaders) > 0 { + headers = strings.Split(string(requestHeaders), ", ") + } + + for _, rule := range corsConfig.CORSRules { + for _, o := range rule.AllowedOrigins { + if o == string(origin) || o == wildcard { + for _, m := range rule.AllowedMethods { + if m == string(method) { + if !checkSubslice(rule.AllowedHeaders, headers) { + continue + } + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + if headers != nil { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, string(requestHeaders)) + } + if rule.ExposeHeaders != nil { + c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(rule.ExposeHeaders, ", ")) + } + if rule.MaxAgeSeconds > 0 || rule.MaxAgeSeconds == -1 { + c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(rule.MaxAgeSeconds)) + } + if o != wildcard { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + } + return + } + } + } + } + } + log.Error(logs.CORSRuleWasNotMatched, logs.TagField(logs.TagDatapath)) + ResponseError(c, "Forbidden", fasthttp.StatusForbidden) +} + +func (h *Handler) SetCORSHeaders(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.SetCORSHeaders") + defer span.End() + + origin := c.Request.Header.Peek(fasthttp.HeaderOrigin) + if len(origin) == 0 { + return + } + + ctx = qostagging.ContextWithIOTag(ctx, internalIOTag) + cidParam, _ := c.UserValue("cid").(string) + reqLog := utils.GetReqLogOrDefault(ctx, h.log) + log := reqLog.With(zap.String("cid", cidParam)) + + corsRule := h.config.CORS() + if corsRule != nil { + setCORSHeadersFromRule(c, corsRule) + return + } + + corsConfig, err := h.getCORSConfig(ctx, log, cidParam) + if err != nil { + log.Error(logs.CouldNotGetCORSConfiguration, zap.Error(err), logs.TagField(logs.TagDatapath)) + return + } + + var withCredentials bool + if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil { + withCredentials = true + } + + for _, rule := range corsConfig.CORSRules { + for _, o := range rule.AllowedOrigins { + if o == string(origin) { + for _, m := range rule.AllowedMethods { + if m == string(c.Method()) { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + c.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) + return + } + } + } + if o == wildcard { + for _, m := range rule.AllowedMethods { + if m == string(c.Method()) { + if withCredentials { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + c.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) + } else { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, o) + } + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + return + } + } + } + } + } +} + +func (h *Handler) getCORSConfig(ctx context.Context, log *zap.Logger, cidStr string) (*data.CORSConfiguration, error) { + cnrID, err := h.resolveContainer(ctx, cidStr) + if err != nil { + return nil, fmt.Errorf("resolve container '%s': %w", cidStr, err) + } + + if cors := h.corsCache.Get(*cnrID); cors != nil { + return cors, nil + } + + objID, err := h.getLastCORSObject(ctx, *cnrID) + if err != nil { + return nil, fmt.Errorf("get last cors object: %w", err) + } + + var addr oid.Address + addr.SetContainer(h.corsCnrID) + addr.SetObject(objID) + corsObj, err := h.frostfs.GetObject(ctx, PrmObjectGet{ + PrmAuth: PrmAuth{ + BearerToken: bearerToken(ctx), + }, + Address: addr, + }) + if err != nil { + return nil, fmt.Errorf("get cors object '%s': %w", addr.EncodeToString(), err) + } + + corsConfig := &data.CORSConfiguration{} + if err = xml.NewDecoder(corsObj.Payload).Decode(corsConfig); err != nil { + return nil, fmt.Errorf("decode cors object: %w", err) + } + + if err = h.corsCache.Put(*cnrID, corsConfig); err != nil { + log.Warn(logs.CouldntCacheCors, zap.Error(err), logs.TagField(logs.TagDatapath)) + } + + return corsConfig, nil +} + +func (h *Handler) getLastCORSObject(ctx context.Context, cnrID cid.ID) (oid.ID, error) { + filters := object.NewSearchFilters() + filters.AddRootFilter() + filters.AddFilter(object.AttributeFilePath, fmt.Sprintf(corsFilePathTemplate, cnrID), object.MatchStringEqual) + + prmAuth := PrmAuth{ + BearerToken: bearerToken(ctx), + } + res, err := h.frostfs.SearchObjects(ctx, PrmObjectSearch{ + PrmAuth: prmAuth, + Container: h.corsCnrID, + Filters: filters, + }) + if err != nil { + return oid.ID{}, fmt.Errorf("search cors versions: %w", err) + } + defer res.Close() + + var ( + addr oid.Address + obj *object.Object + headErr error + objs = make([]*object.Object, 0) + ) + addr.SetContainer(h.corsCnrID) + err = res.Iterate(func(id oid.ID) bool { + addr.SetObject(id) + obj, headErr = h.frostfs.HeadObject(ctx, PrmObjectHead{ + PrmAuth: prmAuth, + Address: addr, + }) + if headErr != nil { + headErr = fmt.Errorf("head cors object '%s': %w", addr.EncodeToString(), headErr) + return true + } + + objs = append(objs, obj) + return false + }) + if err != nil { + return oid.ID{}, fmt.Errorf("iterate cors objects: %w", err) + } + + if headErr != nil { + return oid.ID{}, headErr + } + + if len(objs) == 0 { + return oid.ID{}, errNoCORS + } + + sort.Slice(objs, func(i, j int) bool { + versionID1, _ := objs[i].ID() + versionID2, _ := objs[j].ID() + timestamp1 := utils.GetAttributeValue(objs[i].Attributes(), object.AttributeTimestamp) + timestamp2 := utils.GetAttributeValue(objs[j].Attributes(), object.AttributeTimestamp) + + if objs[i].CreationEpoch() != objs[j].CreationEpoch() { + return objs[i].CreationEpoch() < objs[j].CreationEpoch() + } + + if len(timestamp1) > 0 && len(timestamp2) > 0 && timestamp1 != timestamp2 { + unixTime1, err := strconv.ParseInt(timestamp1, 10, 64) + if err != nil { + return versionID1.EncodeToString() < versionID2.EncodeToString() + } + + unixTime2, err := strconv.ParseInt(timestamp2, 10, 64) + if err != nil { + return versionID1.EncodeToString() < versionID2.EncodeToString() + } + + return unixTime1 < unixTime2 + } + + return versionID1.EncodeToString() < versionID2.EncodeToString() + }) + + objID, _ := objs[len(objs)-1].ID() + return objID, nil +} + +func setCORSHeadersFromRule(c *fasthttp.RequestCtx, cors *data.CORSRule) { + c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(cors.MaxAgeSeconds)) + + if len(cors.AllowedOrigins) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, cors.AllowedOrigins[0]) + } + + if len(cors.AllowedMethods) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(cors.AllowedMethods, ", ")) + } + + if len(cors.AllowedHeaders) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, strings.Join(cors.AllowedHeaders, ", ")) + } + + if len(cors.ExposeHeaders) != 0 { + c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(cors.ExposeHeaders, ", ")) + } + + if cors.AllowedCredentials { + c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + } +} + +func checkSubslice(slice []string, subSlice []string) bool { + if sliceContains(slice, wildcard) { + return true + } + if len(subSlice) > len(slice) { + return false + } + for _, r := range subSlice { + if !sliceContains(slice, r) { + return false + } + } + return true +} + +func sliceContains(slice []string, str string) bool { + for _, s := range slice { + if s == str { + return true + } + } + return false +} diff --git a/internal/handler/cors_test.go b/internal/handler/cors_test.go new file mode 100644 index 0000000..7cd7b0d --- /dev/null +++ b/internal/handler/cors_test.go @@ -0,0 +1,440 @@ +package handler + +import ( + "encoding/base64" + "encoding/xml" + "fmt" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" + "github.com/valyala/fasthttp" +) + +func TestPreflight(t *testing.T) { + hc := prepareHandlerContext(t) + + bktName := "bucket-preflight" + cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private) + require.NoError(t, err) + hc.frostfs.SetContainer(cnrID, cnr) + + var epoch uint64 + + t.Run("CORS object", func(t *testing.T) { + for _, tc := range []struct { + name string + corsConfig *data.CORSConfiguration + requestHeaders map[string]string + expectedHeaders map[string]string + status int + }{ + { + name: "no CORS configuration", + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + fasthttp.HeaderAccessControlExposeHeaders: "", + fasthttp.HeaderAccessControlMaxAge: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + }, + status: fasthttp.StatusNotFound, + }, + { + name: "specific allowed origin", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"http://example.com"}, + AllowedMethods: []string{"GET", "HEAD"}, + AllowedHeaders: []string{"Content-Type"}, + ExposeHeaders: []string{"x-amz-*", "X-Amz-*"}, + MaxAgeSeconds: 900, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "Content-Type", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD", + fasthttp.HeaderAccessControlAllowHeaders: "Content-Type", + fasthttp.HeaderAccessControlExposeHeaders: "x-amz-*, X-Amz-*", + fasthttp.HeaderAccessControlMaxAge: "900", + fasthttp.HeaderAccessControlAllowCredentials: "true", + }, + status: fasthttp.StatusOK, + }, + { + name: "wildcard allowed origin", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + AllowedHeaders: []string{"Content-Type"}, + ExposeHeaders: []string{"x-amz-*", "X-Amz-*"}, + MaxAgeSeconds: 900, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD", + fasthttp.HeaderAccessControlAllowHeaders: "", + fasthttp.HeaderAccessControlExposeHeaders: "x-amz-*, X-Amz-*", + fasthttp.HeaderAccessControlMaxAge: "900", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + status: fasthttp.StatusOK, + }, + { + name: "not allowed header", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + AllowedHeaders: []string{"Content-Type"}, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + fasthttp.HeaderAccessControlRequestHeaders: "Authorization", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + fasthttp.HeaderAccessControlExposeHeaders: "", + fasthttp.HeaderAccessControlMaxAge: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + status: fasthttp.StatusForbidden, + }, + { + name: "empty Origin header", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + }, + }, + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + fasthttp.HeaderAccessControlExposeHeaders: "", + fasthttp.HeaderAccessControlMaxAge: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + status: fasthttp.StatusBadRequest, + }, + { + name: "empty Access-Control-Request-Method header", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + fasthttp.HeaderAccessControlExposeHeaders: "", + fasthttp.HeaderAccessControlMaxAge: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + status: fasthttp.StatusBadRequest, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.corsConfig != nil { + epoch++ + setCORSObject(t, hc, cnrID, tc.corsConfig, epoch) + } + + r := prepareCORSRequest(t, bktName, tc.requestHeaders) + hc.Handler().Preflight(r) + + require.Equal(t, tc.status, r.Response.StatusCode()) + for k, v := range tc.expectedHeaders { + require.Equal(t, v, string(r.Response.Header.Peek(k))) + } + }) + } + }) + + t.Run("CORS config", func(t *testing.T) { + hc.cfg.cors = &data.CORSRule{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + AllowedHeaders: []string{"Content-Type", "Content-Encoding"}, + ExposeHeaders: []string{"x-amz-*", "X-Amz-*"}, + MaxAgeSeconds: 900, + AllowedCredentials: true, + } + + r := prepareCORSRequest(t, bktName, map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }) + hc.Handler().Preflight(r) + + require.Equal(t, fasthttp.StatusOK, r.Response.StatusCode()) + require.Equal(t, "900", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlMaxAge))) + require.Equal(t, "*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowOrigin))) + require.Equal(t, "GET, HEAD", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowMethods))) + require.Equal(t, "Content-Type, Content-Encoding", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowHeaders))) + require.Equal(t, "x-amz-*, X-Amz-*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlExposeHeaders))) + require.Equal(t, "true", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowCredentials))) + }) +} + +func TestSetCORSHeaders(t *testing.T) { + hc := prepareHandlerContext(t) + + bktName := "bucket-set-cors-headers" + cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private) + require.NoError(t, err) + hc.frostfs.SetContainer(cnrID, cnr) + + var epoch uint64 + + t.Run("CORS object", func(t *testing.T) { + for _, tc := range []struct { + name string + corsConfig *data.CORSConfiguration + requestHeaders map[string]string + expectedHeaders map[string]string + }{ + { + name: "empty Origin header", + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderVary: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + }, + { + name: "no CORS configuration", + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderVary: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + }, + }, + { + name: "specific allowed origin", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"http://example.com"}, + AllowedMethods: []string{"GET", "HEAD"}, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD", + fasthttp.HeaderVary: fasthttp.HeaderOrigin, + fasthttp.HeaderAccessControlAllowCredentials: "true", + }, + }, + { + name: "wildcard allowed origin, with credentials", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + }, + }, + }, + requestHeaders: func() map[string]string { + tkn := new(bearer.Token) + err = tkn.Sign(hc.key.PrivateKey) + require.NoError(t, err) + + t64 := base64.StdEncoding.EncodeToString(tkn.Marshal()) + require.NotEmpty(t, t64) + + return map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + fasthttp.HeaderAuthorization: "Bearer " + t64, + } + }(), + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD", + fasthttp.HeaderVary: fasthttp.HeaderOrigin, + fasthttp.HeaderAccessControlAllowCredentials: "true", + }, + }, + { + name: "wildcard allowed origin, without credentials", + corsConfig: &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + }, + }, + }, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://example.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "*", + fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD", + fasthttp.HeaderVary: "", + fasthttp.HeaderAccessControlAllowCredentials: "", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + epoch++ + setCORSObject(t, hc, cnrID, tc.corsConfig, epoch) + r := prepareCORSRequest(t, bktName, tc.requestHeaders) + hc.Handler().SetCORSHeaders(r) + + require.Equal(t, fasthttp.StatusOK, r.Response.StatusCode()) + for k, v := range tc.expectedHeaders { + require.Equal(t, v, string(r.Response.Header.Peek(k))) + } + }) + } + }) + + t.Run("CORS config", func(t *testing.T) { + hc.cfg.cors = &data.CORSRule{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "HEAD"}, + AllowedHeaders: []string{"Content-Type", "Content-Encoding"}, + ExposeHeaders: []string{"x-amz-*", "X-Amz-*"}, + MaxAgeSeconds: 900, + AllowedCredentials: true, + } + + r := prepareCORSRequest(t, bktName, map[string]string{fasthttp.HeaderOrigin: "http://example.com"}) + hc.Handler().SetCORSHeaders(r) + + require.Equal(t, "900", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlMaxAge))) + require.Equal(t, "*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowOrigin))) + require.Equal(t, "GET, HEAD", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowMethods))) + require.Equal(t, "Content-Type, Content-Encoding", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowHeaders))) + require.Equal(t, "x-amz-*, X-Amz-*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlExposeHeaders))) + require.Equal(t, "true", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowCredentials))) + }) +} + +func TestCheckSubslice(t *testing.T) { + for _, tc := range []struct { + name string + allowed []string + actual []string + expected bool + }{ + { + name: "empty allowed slice", + allowed: []string{}, + actual: []string{"str1", "str2", "str3"}, + expected: false, + }, + { + name: "empty actual slice", + allowed: []string{"str1", "str2", "str3"}, + actual: []string{}, + expected: true, + }, + { + name: "allowed wildcard", + allowed: []string{"str", "*"}, + actual: []string{"str1", "str2", "str3"}, + expected: true, + }, + { + name: "similar allowed and actual", + allowed: []string{"str1", "str2", "str3"}, + actual: []string{"str1", "str2", "str3"}, + expected: true, + }, + { + name: "allowed actual", + allowed: []string{"str", "str1", "str2", "str4"}, + actual: []string{"str1", "str2"}, + expected: true, + }, + { + name: "not allowed actual", + allowed: []string{"str", "str1", "str2", "str4"}, + actual: []string{"str1", "str5"}, + expected: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, checkSubslice(tc.allowed, tc.actual)) + }) + } +} + +func setCORSObject(t *testing.T, hc *handlerContext, cnrID cid.ID, corsConfig *data.CORSConfiguration, epoch uint64) { + payload, err := xml.Marshal(corsConfig) + require.NoError(t, err) + + a := object.NewAttribute() + a.SetKey(object.AttributeFilePath) + a.SetValue(fmt.Sprintf(corsFilePathTemplate, cnrID)) + + objID := oidtest.ID() + obj := object.New() + obj.SetAttributes(*a) + obj.SetOwnerID(hc.owner) + obj.SetPayload(payload) + obj.SetPayloadSize(uint64(len(payload))) + obj.SetContainerID(hc.corsCnr) + obj.SetID(objID) + obj.SetCreationEpoch(epoch) + + var addr oid.Address + addr.SetObject(objID) + addr.SetContainer(hc.corsCnr) + + hc.frostfs.SetObject(addr, obj) +} diff --git a/internal/handler/frostfs_mock.go b/internal/handler/frostfs_mock.go index b60915e..7d72ad9 100644 --- a/internal/handler/frostfs_mock.go +++ b/internal/handler/frostfs_mock.go @@ -52,6 +52,10 @@ func (t *TestFrostFS) SetContainer(cnrID cid.ID, cnr *container.Container) { t.containers[cnrID.EncodeToString()] = cnr } +func (t *TestFrostFS) SetObject(addr oid.Address, obj *object.Object) { + t.objects[addr.EncodeToString()] = obj +} + // AllowUserOperation grants access to object operations. // Empty userID and objID means any user and object respectively. func (t *TestFrostFS) AllowUserOperation(cnrID cid.ID, userID user.ID, op acl.Op, objID oid.ID) { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 179cf60..48f8f55 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -36,6 +36,8 @@ type Config interface { BufferMaxSizeForPut() uint64 NamespaceHeader() string EnableFilepathFallback() bool + FormContainerZone(string) string + CORS() *data.CORSRule } // PrmContainer groups parameters of FrostFS.Container operation. @@ -158,7 +160,7 @@ type FrostFS interface { } type ContainerResolver interface { - Resolve(ctx context.Context, name string) (*cid.ID, error) + Resolve(ctx context.Context, zone, name string) (*cid.ID, error) } type Handler struct { @@ -170,14 +172,18 @@ type Handler struct { tree layer.TreeService cache *cache.BucketCache workerPool *ants.Pool + corsCnrID cid.ID + corsCache *cache.CORSCache } type AppParams struct { - Logger *zap.Logger - FrostFS FrostFS - Owner *user.ID - Resolver ContainerResolver - Cache *cache.BucketCache + Logger *zap.Logger + FrostFS FrostFS + Owner *user.ID + Resolver ContainerResolver + Cache *cache.BucketCache + CORSCnrID cid.ID + CORSCache *cache.CORSCache } func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler { @@ -190,6 +196,8 @@ func New(params *AppParams, config Config, tree layer.TreeService, workerPool *a tree: tree, cache: params.Cache, workerPool: workerPool, + corsCnrID: params.CORSCnrID, + corsCache: params.CORSCache, } } @@ -352,7 +360,14 @@ func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*ci cnrID := new(cid.ID) err := cnrID.DecodeString(containerID) if err != nil { - cnrID, err = h.containerResolver.Resolve(ctx, containerID) + var namespace string + namespace, err = middleware.GetNamespace(ctx) + if err != nil { + return nil, err + } + + zone := h.config.FormContainerZone(namespace) + cnrID, err = h.containerResolver.Resolve(ctx, zone, containerID) if err != nil && strings.Contains(err.Error(), "not found") { err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error()) } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index ab2cd9f..3a81c50 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -16,7 +16,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -61,6 +63,7 @@ func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*d type configMock struct { additionalSearch bool + cors *data.CORSRule } func (c *configMock) DefaultTimestamp() bool { @@ -99,9 +102,18 @@ func (c *configMock) EnableFilepathFallback() bool { return c.additionalSearch } +func (c *configMock) FormContainerZone(string) string { + return v2container.SysAttributeZoneDefault +} + +func (c *configMock) CORS() *data.CORSRule { + return c.cors +} + type handlerContext struct { - key *keys.PrivateKey - owner user.ID + key *keys.PrivateKey + owner user.ID + corsCnr cid.ID h *Handler frostfs *TestFrostFS @@ -131,10 +143,12 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { testFrostFS := NewTestFrostFS(key) testResolver := &resolver.Resolver{Name: "test_resolver"} - testResolver.SetResolveFunc(func(_ context.Context, name string) (*cid.ID, error) { + testResolver.SetResolveFunc(func(_ context.Context, _, name string) (*cid.ID, error) { return testFrostFS.ContainerID(name) }) + cnrID := createCORSContainer(owner, testFrostFS) + params := &AppParams{ Logger: logger, FrostFS: testFrostFS, @@ -145,6 +159,12 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { Lifetime: 1, Logger: logger, }, false), + CORSCnrID: cnrID, + CORSCache: cache.NewCORSCache(&cache.Config{ + Size: 1, + Lifetime: 1, + Logger: logger, + }), } treeMock := newTreeService() @@ -159,6 +179,7 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { return &handlerContext{ key: key, owner: owner, + corsCnr: cnrID, h: handler, frostfs: testFrostFS, tree: treeMock, @@ -166,6 +187,20 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { }, nil } +func createCORSContainer(owner user.ID, frostfs *TestFrostFS) cid.ID { + var cnr container.Container + cnr.Init() + cnr.SetOwner(owner) + + cnrID := cidtest.ID() + frostfs.SetContainer(cnrID, &cnr) + frostfs.AllowUserOperation(cnrID, owner, acl.OpObjectSearch, oid.ID{}) + frostfs.AllowUserOperation(cnrID, owner, acl.OpObjectHead, oid.ID{}) + frostfs.AllowUserOperation(cnrID, owner, acl.OpObjectGet, oid.ID{}) + + return cnrID +} + func (hc *handlerContext) prepareContainer(name string, basicACL acl.Basic) (cid.ID, *container.Container, error) { var pp netmap.PlacementPolicy err := pp.DecodeString("REP 1") @@ -486,6 +521,25 @@ func prepareGetRequest(ctx context.Context, bucket, objID string) *fasthttp.Requ return r } +func prepareCORSRequest(t *testing.T, bucket string, headers map[string]string) *fasthttp.RequestCtx { + ctx := context.Background() + ctx = middleware.SetNamespace(ctx, "") + + r := new(fasthttp.RequestCtx) + r.SetUserValue("cid", bucket) + + for k, v := range headers { + r.Request.Header.Set(k, v) + } + + ctx, err := tokens.StoreBearerTokenAppCtx(ctx, r) + require.NoError(t, err) + + utils.SetContextToRequest(ctx, r) + + return r +} + func prepareGetByAttributeRequest(ctx context.Context, bucket, attrKey, attrVal string) *fasthttp.RequestCtx { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index f8f1da9..3166f98 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -72,41 +72,47 @@ const ( TagsLogConfigWontBeUpdated = "tags log config won't be updated" FailedToReadIndexPageTemplate = "failed to read index page template" SetCustomIndexPageTemplate = "set custom index page template" + CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info" ) // Log messages with the "datapath" tag. const ( - CouldntParseCreationDate = "couldn't parse creation date" - CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" - FailedToAddObjectToArchive = "failed to add object to archive" - CloseZipWriter = "close zip writer" - IgnorePartEmptyFormName = "ignore part, empty form name" - IgnorePartEmptyFilename = "ignore part, empty filename" - CouldNotParseClientTime = "could not parse client time" - CouldNotPrepareExpirationHeader = "could not prepare expiration header" - CouldNotEncodeResponse = "could not encode response" - AddAttributeToResultObject = "add attribute to result object" - Request = "request" - CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" - CouldntPutBucketIntoCache = "couldn't put bucket info into cache" - FailedToIterateOverResponse = "failed to iterate over search response" - InvalidCacheEntryType = "invalid cache entry type" - FailedToUnescapeQuery = "failed to unescape query" - CouldntCacheNetmap = "couldn't cache netmap" - FailedToCloseReader = "failed to close reader" - FailedToFilterHeaders = "failed to filter headers" - FailedToReadFileFromTar = "failed to read file from tar" - FailedToGetAttributes = "failed to get attributes" - CloseGzipWriter = "close gzip writer" - CloseTarWriter = "close tar writer" - FailedToCreateGzipReader = "failed to create gzip reader" - GzipReaderSelected = "gzip reader selected" - CouldNotReceiveMultipartForm = "could not receive multipart/form" - ObjectsNotFound = "objects not found" - IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" - CouldNotGetBucket = "could not get bucket" - CouldNotResolveContainerID = "could not resolve container id" - FailedToSumbitTaskToPool = "failed to submit task to pool" + CouldntParseCreationDate = "couldn't parse creation date" + CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" + FailedToAddObjectToArchive = "failed to add object to archive" + CloseZipWriter = "close zip writer" + IgnorePartEmptyFormName = "ignore part, empty form name" + IgnorePartEmptyFilename = "ignore part, empty filename" + CouldNotParseClientTime = "could not parse client time" + CouldNotPrepareExpirationHeader = "could not prepare expiration header" + CouldNotEncodeResponse = "could not encode response" + AddAttributeToResultObject = "add attribute to result object" + Request = "request" + CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" + CouldntPutBucketIntoCache = "couldn't put bucket info into cache" + FailedToIterateOverResponse = "failed to iterate over search response" + InvalidCacheEntryType = "invalid cache entry type" + FailedToUnescapeQuery = "failed to unescape query" + CouldntCacheNetmap = "couldn't cache netmap" + FailedToCloseReader = "failed to close reader" + FailedToFilterHeaders = "failed to filter headers" + FailedToReadFileFromTar = "failed to read file from tar" + FailedToGetAttributes = "failed to get attributes" + CloseGzipWriter = "close gzip writer" + CloseTarWriter = "close tar writer" + FailedToCreateGzipReader = "failed to create gzip reader" + GzipReaderSelected = "gzip reader selected" + CouldNotReceiveMultipartForm = "could not receive multipart/form" + ObjectsNotFound = "objects not found" + IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" + CouldNotGetBucket = "could not get bucket" + CouldNotResolveContainerID = "could not resolve container id" + FailedToSumbitTaskToPool = "failed to submit task to pool" + CouldNotGetCORSConfiguration = "could not get cors configuration" + EmptyOriginRequestHeader = "empty Origin request header" + EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header" + CORSRuleWasNotMatched = "cors rule was not matched" + CouldntCacheCors = "couldn't cache cors" ) // Log messages with the "external_storage" tag. diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index 9115930..4cf45a4 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -10,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -21,8 +20,6 @@ import ( "google.golang.org/grpc/status" ) -const clientIOTag = "client" - // FrostFS represents virtual connection to the FrostFS network. // It is used to provide an interface to dependent packages // which work with FrostFS. @@ -70,7 +67,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) prmPut.UseBearer(*prm.BearerToken) } - idObj, err := x.pool.PutObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmPut) + idObj, err := x.pool.PutObject(ctx, prmPut) if err != nil { return oid.ID{}, handleObjectError("save object via connection pool", err) } @@ -103,7 +100,7 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o prmHead.UseBearer(*prm.BearerToken) } - res, err := x.pool.HeadObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmHead) + res, err := x.pool.HeadObject(ctx, prmHead) if err != nil { return nil, handleObjectError("read object header via connection pool", err) } @@ -123,7 +120,7 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han prmGet.UseBearer(*prm.BearerToken) } - res, err := x.pool.GetObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmGet) + res, err := x.pool.GetObject(ctx, prmGet) if err != nil { return nil, handleObjectError("init full object reading via connection pool", err) } @@ -148,7 +145,7 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) ( prmRange.UseBearer(*prm.BearerToken) } - res, err := x.pool.ObjectRange(qostagging.ContextWithIOTag(ctx, clientIOTag), prmRange) + res, err := x.pool.ObjectRange(ctx, prmRange) if err != nil { return nil, handleObjectError("init payload range reading via connection pool", err) } @@ -169,7 +166,7 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch prmSearch.UseBearer(*prm.BearerToken) } - res, err := x.pool.SearchObjects(qostagging.ContextWithIOTag(ctx, clientIOTag), prmSearch) + res, err := x.pool.SearchObjects(ctx, prmSearch) if err != nil { return nil, handleObjectError("init object search via connection pool", err) } diff --git a/internal/service/frostfs/tree_pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go index 89afc3c..410acda 100644 --- a/internal/service/frostfs/tree_pool_wrapper.go +++ b/internal/service/frostfs/tree_pool_wrapper.go @@ -10,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" apitree "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/tree" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" ) @@ -62,7 +61,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([ BearerToken: getBearer(ctx), } - nodes, err := w.p.GetNodes(qostagging.ContextWithIOTag(ctx, clientIOTag), poolPrm) + nodes, err := w.p.GetNodes(ctx, poolPrm) if err != nil { return nil, handleError(err) } @@ -121,7 +120,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, poolPrm.RootID = nil } - subTreeReader, err := w.p.GetSubTree(qostagging.ContextWithIOTag(ctx, clientIOTag), poolPrm) + subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) if err != nil { return nil, handleError(err) } diff --git a/resolver/resolver.go b/resolver/resolver.go index e7615d4..6d7c5d5 100644 --- a/resolver/resolver.go +++ b/resolver/resolver.go @@ -6,7 +6,7 @@ import ( "fmt" "sync" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns" @@ -29,14 +29,9 @@ type FrostFS interface { SystemDNS(context.Context) (string, error) } -type Settings interface { - FormContainerZone(ns string) (zone string, isDefault bool) -} - type Config struct { FrostFS FrostFS RPCAddress string - Settings Settings } type ContainerResolver struct { @@ -46,15 +41,15 @@ type ContainerResolver struct { type Resolver struct { Name string - resolve func(context.Context, string) (*cid.ID, error) + resolve func(context.Context, string, string) (*cid.ID, error) } -func (r *Resolver) SetResolveFunc(fn func(context.Context, string) (*cid.ID, error)) { +func (r *Resolver) SetResolveFunc(fn func(context.Context, string, string) (*cid.ID, error)) { r.resolve = fn } -func (r *Resolver) Resolve(ctx context.Context, name string) (*cid.ID, error) { - return r.resolve(ctx, name) +func (r *Resolver) Resolve(ctx context.Context, zone, name string) (*cid.ID, error) { + return r.resolve(ctx, zone, name) } func NewContainerResolver(resolverNames []string, cfg *Config) (*ContainerResolver, error) { @@ -81,13 +76,13 @@ func createResolvers(resolverNames []string, cfg *Config) ([]*Resolver, error) { return resolvers, nil } -func (r *ContainerResolver) Resolve(ctx context.Context, cnrName string) (*cid.ID, error) { +func (r *ContainerResolver) Resolve(ctx context.Context, cnrZone, cnrName string) (*cid.ID, error) { r.mu.RLock() defer r.mu.RUnlock() var err error for _, resolver := range r.resolvers { - cnrID, resolverErr := resolver.Resolve(ctx, cnrName) + cnrID, resolverErr := resolver.Resolve(ctx, cnrZone, cnrName) if resolverErr != nil { resolverErr = fmt.Errorf("%s: %w", resolver.Name, resolverErr) if err == nil { @@ -141,34 +136,25 @@ func (r *ContainerResolver) equals(resolverNames []string) bool { func newResolver(name string, cfg *Config) (*Resolver, error) { switch name { case DNSResolver: - return NewDNSResolver(cfg.FrostFS, cfg.Settings) + return NewDNSResolver(cfg.FrostFS) case NNSResolver: - return NewNNSResolver(cfg.RPCAddress, cfg.Settings) + return NewNNSResolver(cfg.RPCAddress) default: return nil, fmt.Errorf("unknown resolver: %s", name) } } -func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) { +func NewDNSResolver(frostFS FrostFS) (*Resolver, error) { if frostFS == nil { return nil, fmt.Errorf("pool must not be nil for DNS resolver") } - if settings == nil { - return nil, fmt.Errorf("resolver settings must not be nil for DNS resolver") - } var dns ns.DNS - resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) { + resolveFunc := func(ctx context.Context, zone, name string) (*cid.ID, error) { var err error - namespace, err := middleware.GetNamespace(ctx) - if err != nil { - return nil, err - } - - zone, isDefault := settings.FormContainerZone(namespace) - if isDefault { + if zone == v2container.SysAttributeZoneDefault { zone, err = frostFS.SystemDNS(ctx) if err != nil { return nil, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err) @@ -190,13 +176,10 @@ func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) { }, nil } -func NewNNSResolver(rpcAddress string, settings Settings) (*Resolver, error) { +func NewNNSResolver(rpcAddress string) (*Resolver, error) { if rpcAddress == "" { return nil, fmt.Errorf("rpc address must not be empty for NNS resolver") } - if settings == nil { - return nil, fmt.Errorf("resolver settings must not be nil for NNS resolver") - } var nns ns.NNS @@ -204,16 +187,9 @@ func NewNNSResolver(rpcAddress string, settings Settings) (*Resolver, error) { return nil, fmt.Errorf("could not dial nns: %w", err) } - resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) { + resolveFunc := func(_ context.Context, zone, name string) (*cid.ID, error) { var d container.Domain d.SetName(name) - - namespace, err := middleware.GetNamespace(ctx) - if err != nil { - return nil, err - } - - zone, _ := settings.FormContainerZone(namespace) d.SetZone(zone) cnrID, err := nns.ResolveContainerDomain(d) diff --git a/utils/attributes.go b/utils/attributes.go index 4d277a9..55fadaa 100644 --- a/utils/attributes.go +++ b/utils/attributes.go @@ -11,6 +11,8 @@ import ( "time" "unicode" "unicode/utf8" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) type EpochDurations struct { @@ -256,3 +258,12 @@ func (t systemTransformer) updateExpirationHeader(headers map[string]string, dur headers[t.expirationEpochAttr()] = strconv.FormatUint(expirationEpoch, 10) } + +func GetAttributeValue(attrs []object.Attribute, key string) string { + for _, attr := range attrs { + if attr.Key() == key { + return attr.Value() + } + } + return "" +} From d670983df4453b91fa96cc0ab5400d6e7e89fcd2 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Mon, 10 Feb 2025 18:30:16 +0300 Subject: [PATCH 534/548] [#208] govulncheck: Fix minor toolchain updates for good Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/vulncheck.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 5cb6e73..5fb9dc5 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -16,7 +16,8 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22.12' + go-version: '1.22' + check-latest: true - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest From 0f73da258bb0974097a2cf86242cc1c10ddd4198 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 20 Mar 2025 18:39:30 +0300 Subject: [PATCH 535/548] [#223] Bump frostfs-sdk-go Contains: * more detailed pool errors * disabled service config query in gRPC client Signed-off-by: Alex Vanin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0ace5f2..31cf242 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 diff --git a/go.sum b/go.sum index a2121ab..6050ad6 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,8 @@ git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe h1:81gDNdWNLP24oMQukRiCE9R1wGSh0l0dRq3F1W+Oesc= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc h1:fS6Yp4GvI+C22UrWz9oqJXwvQw5Q6SmADIY4H9eIQsc= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 458bf933fcef2f782d91d60c70fd5b89a64ff859 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 3 Mar 2025 18:06:41 +0300 Subject: [PATCH 536/548] [#191] Refactor error handling and logging Signed-off-by: Denis Kirillov --- cmd/http-gw/app.go | 4 +- docs/api.md | 26 ++-- internal/handler/browse.go | 22 ++- internal/handler/cors.go | 70 ++++----- internal/handler/download.go | 133 ++++++++---------- internal/handler/handler.go | 107 ++++++-------- internal/handler/handler_test.go | 2 +- internal/handler/head.go | 40 +++--- internal/handler/multipart.go | 4 +- internal/handler/reader.go | 27 ++-- internal/handler/upload.go | 109 ++++++-------- internal/handler/utils.go | 83 ++++------- internal/logs/logs.go | 30 ++-- internal/service/frostfs/frostfs.go | 29 ++-- internal/service/frostfs/frostfs_test.go | 14 +- internal/service/frostfs/tree_pool_wrapper.go | 8 +- tree/tree.go | 17 ++- 17 files changed, 327 insertions(+), 398 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index c75f9d8..de186fb 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -724,12 +724,12 @@ func (a *app) stopServices() { } func (a *app) configureRouter(workerPool *ants.Pool) { - a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) + a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool) r := router.New() r.RedirectTrailingSlash = true r.NotFound = func(r *fasthttp.RequestCtx) { - handler.ResponseError(r, "Not found", fasthttp.StatusNotFound) + handler.ResponseError(r, "Route Not found", fasthttp.StatusNotFound) } r.MethodNotAllowed = func(r *fasthttp.RequestCtx) { handler.ResponseError(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) diff --git a/docs/api.md b/docs/api.md index d099915..698e9b1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -94,6 +94,8 @@ The `filename` field from the multipart form will be set as `FileName` attribute |--------|----------------------------------------------| | 200 | Object created successfully. | | 400 | Some error occurred during object uploading. | +| 403 | Access denied. | +| 409 | Can not upload object due to quota reached. | ## Get object @@ -141,6 +143,7 @@ Get an object (payload and attributes) by an address. |--------|------------------------------------------------| | 200 | Object got successfully. | | 400 | Some error occurred during object downloading. | +| 403 | Access denied. | | 404 | Container or object not found. | ###### Body @@ -183,6 +186,7 @@ Get an object attributes by an address. |--------|---------------------------------------------------| | 200 | Object head successfully. | | 400 | Some error occurred during object HEAD operation. | +| 403 | Access denied. | | 404 | Container or object not found. | ## Search object @@ -233,6 +237,7 @@ If more than one object is found, an arbitrary one will be returned. |--------|------------------------------------------------| | 200 | Object got successfully. | | 400 | Some error occurred during object downloading. | +| 403 | Access denied. | | 404 | Container or object not found. | #### HEAD @@ -269,6 +274,7 @@ If more than one object is found, an arbitrary one will be used to get attribute |--------|---------------------------------------| | 200 | Object head successfully. | | 400 | Some error occurred during operation. | +| 403 | Access denied. | | 404 | Container or object not found. | ## Download archive @@ -304,16 +310,16 @@ Archive can be compressed (see http-gw [configuration](gate-configuration.md#arc ###### Headers -| Header | Description | -|-----------------------|-------------------------------------------------------------------------------------------------------------------| -| `Content-Disposition` | Indicate how to browsers should treat file (`attachment`). Set `filename` as `archive.zip`. | -| `Content-Type` | Indicate content type of object. Set to `application/zip` | +| Header | Description | +|-----------------------|---------------------------------------------------------------------------------------------| +| `Content-Disposition` | Indicate how to browsers should treat file (`attachment`). Set `filename` as `archive.zip`. | +| `Content-Type` | Indicate content type of object. Set to `application/zip` | ###### Status codes -| Status | Description | -|--------|-----------------------------------------------------| -| 200 | Object got successfully. | -| 400 | Some error occurred during object downloading. | -| 404 | Container or objects not found. | -| 500 | Some inner error (e.g. error on streaming objects). | +| Status | Description | +|--------|------------------------------------------------| +| 200 | Object got successfully. | +| 400 | Some error occurred during object downloading. | +| 403 | Access denied. | +| 404 | Container or objects not found. | diff --git a/internal/handler/browse.go b/internal/handler/browse.go index 2d0e34d..ebe9004 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -223,7 +223,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck return nil, err } - log := utils.GetReqLogOrDefault(ctx, h.log) + log := h.reqLogger(ctx) dirs := make(map[string]struct{}) result := &GetObjectsResponse{ objects: make([]ResponseObject, 0, 100), @@ -258,7 +258,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re go func() { defer close(res) - log := utils.GetReqLogOrDefault(ctx, h.log).With( + log := h.reqLogger(ctx).With( zap.String("cid", cnrID.EncodeToString()), zap.String("path", basePath), ) @@ -273,7 +273,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re }) if err != nil { wg.Done() - log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath)) + log.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath)) } select { case <-ctx.Done(): @@ -328,20 +328,18 @@ type browseParams struct { listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) (*GetObjectsResponse, error) } -func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) { +func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p browseParams) { const S3Protocol = "s3" const FrostfsProtocol = "frostfs" - ctx := utils.GetContextFromRequest(c) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With( + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( zap.String("bucket", p.bucketInfo.Name), zap.String("container", p.bucketInfo.CID.EncodeToString()), zap.String("prefix", p.prefix), - ) + )) resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToListObjects, err) return } @@ -360,7 +358,7 @@ func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) { "parentDir": parentDir, }).Parse(h.config.IndexPageTemplate()) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToParseTemplate, err) return } bucketName := p.bucketInfo.Name @@ -369,14 +367,14 @@ func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) { bucketName = p.bucketInfo.CID.EncodeToString() protocol = FrostfsProtocol } - if err = tmpl.Execute(c, &BrowsePageData{ + if err = tmpl.Execute(req, &BrowsePageData{ Container: bucketName, Prefix: p.prefix, Objects: objects, Protocol: protocol, HasErrors: resp.hasErrors, }); err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToExecuteTemplate, err) return } } diff --git a/internal/handler/cors.go b/internal/handler/cors.go index 234ef2a..d77ae02 100644 --- a/internal/handler/cors.go +++ b/internal/handler/cors.go @@ -30,32 +30,32 @@ const ( var errNoCORS = errors.New("no CORS objects found") -func (h *Handler) Preflight(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.Preflight") +func (h *Handler) Preflight(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.Preflight") defer span.End() ctx = qostagging.ContextWithIOTag(ctx, internalIOTag) - cidParam, _ := c.UserValue("cid").(string) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) + cidParam, _ := req.UserValue("cid").(string) + reqLog := h.reqLogger(ctx) log := reqLog.With(zap.String("cid", cidParam)) - origin := c.Request.Header.Peek(fasthttp.HeaderOrigin) + origin := req.Request.Header.Peek(fasthttp.HeaderOrigin) if len(origin) == 0 { log.Error(logs.EmptyOriginRequestHeader, logs.TagField(logs.TagDatapath)) - ResponseError(c, "Origin request header needed", fasthttp.StatusBadRequest) + ResponseError(req, "Origin request header needed", fasthttp.StatusBadRequest) return } - method := c.Request.Header.Peek(fasthttp.HeaderAccessControlRequestMethod) + method := req.Request.Header.Peek(fasthttp.HeaderAccessControlRequestMethod) if len(method) == 0 { log.Error(logs.EmptyAccessControlRequestMethodHeader, logs.TagField(logs.TagDatapath)) - ResponseError(c, "Access-Control-Request-Method request header needed", fasthttp.StatusBadRequest) + ResponseError(req, "Access-Control-Request-Method request header needed", fasthttp.StatusBadRequest) return } corsRule := h.config.CORS() if corsRule != nil { - setCORSHeadersFromRule(c, corsRule) + setCORSHeadersFromRule(req, corsRule) return } @@ -66,12 +66,12 @@ func (h *Handler) Preflight(c *fasthttp.RequestCtx) { if errors.Is(err, errNoCORS) { status = fasthttp.StatusNotFound } - ResponseError(c, "could not get CORS configuration: "+err.Error(), status) + ResponseError(req, "could not get CORS configuration: "+err.Error(), status) return } var headers []string - requestHeaders := c.Request.Header.Peek(fasthttp.HeaderAccessControlRequestHeaders) + requestHeaders := req.Request.Header.Peek(fasthttp.HeaderAccessControlRequestHeaders) if len(requestHeaders) > 0 { headers = strings.Split(string(requestHeaders), ", ") } @@ -84,19 +84,19 @@ func (h *Handler) Preflight(c *fasthttp.RequestCtx) { if !checkSubslice(rule.AllowedHeaders, headers) { continue } - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) if headers != nil { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, string(requestHeaders)) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, string(requestHeaders)) } if rule.ExposeHeaders != nil { - c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(rule.ExposeHeaders, ", ")) + req.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(rule.ExposeHeaders, ", ")) } if rule.MaxAgeSeconds > 0 || rule.MaxAgeSeconds == -1 { - c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(rule.MaxAgeSeconds)) + req.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(rule.MaxAgeSeconds)) } if o != wildcard { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") } return } @@ -105,26 +105,26 @@ func (h *Handler) Preflight(c *fasthttp.RequestCtx) { } } log.Error(logs.CORSRuleWasNotMatched, logs.TagField(logs.TagDatapath)) - ResponseError(c, "Forbidden", fasthttp.StatusForbidden) + ResponseError(req, "Forbidden", fasthttp.StatusForbidden) } -func (h *Handler) SetCORSHeaders(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.SetCORSHeaders") +func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.SetCORSHeaders") defer span.End() - origin := c.Request.Header.Peek(fasthttp.HeaderOrigin) + origin := req.Request.Header.Peek(fasthttp.HeaderOrigin) if len(origin) == 0 { return } ctx = qostagging.ContextWithIOTag(ctx, internalIOTag) - cidParam, _ := c.UserValue("cid").(string) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) + cidParam, _ := req.UserValue("cid").(string) + reqLog := h.reqLogger(ctx) log := reqLog.With(zap.String("cid", cidParam)) corsRule := h.config.CORS() if corsRule != nil { - setCORSHeadersFromRule(c, corsRule) + setCORSHeadersFromRule(req, corsRule) return } @@ -143,26 +143,26 @@ func (h *Handler) SetCORSHeaders(c *fasthttp.RequestCtx) { for _, o := range rule.AllowedOrigins { if o == string(origin) { for _, m := range rule.AllowedMethods { - if m == string(c.Method()) { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") - c.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) + if m == string(req.Method()) { + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + req.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) return } } } if o == wildcard { for _, m := range rule.AllowedMethods { - if m == string(c.Method()) { + if m == string(req.Method()) { if withCredentials { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") - c.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") + req.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin) } else { - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, o) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, o) } - c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) + req.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) return } } diff --git a/internal/handler/download.go b/internal/handler/download.go index b398a54..114bf34 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -25,43 +25,38 @@ import ( ) // DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. -func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadByAddressOrBucketName") +func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.DownloadByAddressOrBucketName") defer span.End() - utils.SetContextToRequest(ctx, c) - cidParam := c.UserValue("cid").(string) - oidParam := c.UserValue("oid").(string) - downloadParam := c.QueryArgs().GetBool("download") + cidParam := req.UserValue("cid").(string) + oidParam := req.UserValue("oid").(string) + downloadParam := req.QueryArgs().GetBool("download") - log := utils.GetReqLogOrDefault(ctx, h.log).With( + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( zap.String("cid", cidParam), zap.String("oid", oidParam), - ) + )) - bktInfo, err := h.getBucketInfo(ctx, cidParam, log) + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { - log.Error(logs.FailedToCheckIfSettingsNodeExist, zap.String("cid", bktInfo.CID.String()), - zap.Error(checkS3Err), logs.TagField(logs.TagExternalStorageTree)) - logAndSendBucketError(c, log, checkS3Err) + h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) return } - req := newRequest(c, log) - var objID oid.ID if checkS3Err == nil && shouldDownload(oidParam, downloadParam) { h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile) } else if err = objID.DecodeString(oidParam); err == nil { h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile) } else { - h.browseIndex(c, checkS3Err != nil) + h.browseIndex(ctx, req, cidParam, oidParam, checkS3Err != nil) } } @@ -70,12 +65,11 @@ func shouldDownload(oidParam string, downloadParam bool) bool { } // DownloadByAttribute handles attribute-based download requests. -func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadByAttribute") +func (h *Handler) DownloadByAttribute(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.DownloadByAttribute") defer span.End() - utils.SetContextToRequest(ctx, c) - h.byAttribute(c, h.receiveFile) + h.byAttribute(ctx, req, h.receiveFile) } func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) { @@ -95,31 +89,33 @@ func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op } // DownloadZip handles zip by prefix requests. -func (h *Handler) DownloadZip(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadZip") +func (h *Handler) DownloadZip(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.DownloadZip") defer span.End() - utils.SetContextToRequest(ctx, c) - scid, _ := c.UserValue("cid").(string) + scid, _ := req.UserValue("cid").(string) + prefix, _ := req.UserValue("prefix").(string) - log := utils.GetReqLogOrDefault(ctx, h.log) - bktInfo, err := h.getBucketInfo(ctx, scid, log) + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(zap.String("cid", scid), zap.String("prefix", prefix))) + + bktInfo, err := h.getBucketInfo(ctx, scid) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } - resSearch, err := h.searchObjectsByPrefix(c, log, bktInfo.CID) + + resSearch, err := h.searchObjectsByPrefix(ctx, bktInfo.CID, prefix) if err != nil { return } - c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") - c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") + req.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") + req.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") - c.SetBodyStreamWriter(h.getZipResponseWriter(ctx, log, resSearch, bktInfo)) + req.SetBodyStreamWriter(h.getZipResponseWriter(ctx, resSearch, bktInfo)) } -func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { +func (h *Handler) getZipResponseWriter(ctx context.Context, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { return func(w *bufio.Writer) { defer resSearch.Close() @@ -127,20 +123,20 @@ func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, res zipWriter := zip.NewWriter(w) var objectsWritten int - errIter := resSearch.Iterate(h.putObjectToArchive(ctx, log, bktInfo.CID, buf, + errIter := resSearch.Iterate(h.putObjectToArchive(ctx, bktInfo.CID, buf, func(obj *object.Object) (io.Writer, error) { objectsWritten++ return h.createZipFile(zipWriter, obj) }), ) if errIter != nil { - log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) return } else if objectsWritten == 0 { - log.Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) } if err := zipWriter.Close(); err != nil { - log.Error(logs.CloseZipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.CloseZipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } } } @@ -164,31 +160,33 @@ func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, } // DownloadTar forms tar.gz from objects by prefix. -func (h *Handler) DownloadTar(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadTar") +func (h *Handler) DownloadTar(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.DownloadTar") defer span.End() - utils.SetContextToRequest(ctx, c) - scid, _ := c.UserValue("cid").(string) + scid, _ := req.UserValue("cid").(string) + prefix, _ := req.UserValue("prefix").(string) - log := utils.GetReqLogOrDefault(ctx, h.log) - bktInfo, err := h.getBucketInfo(ctx, scid, log) + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(zap.String("cid", scid), zap.String("prefix", prefix))) + + bktInfo, err := h.getBucketInfo(ctx, scid) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } - resSearch, err := h.searchObjectsByPrefix(c, log, bktInfo.CID) + + resSearch, err := h.searchObjectsByPrefix(ctx, bktInfo.CID, prefix) if err != nil { return } - c.Response.Header.Set(fasthttp.HeaderContentType, "application/gzip") - c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.tar.gz\"") + req.Response.Header.Set(fasthttp.HeaderContentType, "application/gzip") + req.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.tar.gz\"") - c.SetBodyStreamWriter(h.getTarResponseWriter(ctx, log, resSearch, bktInfo)) + req.SetBodyStreamWriter(h.getTarResponseWriter(ctx, resSearch, bktInfo)) } -func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { +func (h *Handler) getTarResponseWriter(ctx context.Context, resSearch ResObjectSearch, bktInfo *data.BucketInfo) func(w *bufio.Writer) { return func(w *bufio.Writer) { defer resSearch.Close() @@ -203,26 +201,26 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res defer func() { if err := tarWriter.Close(); err != nil { - log.Error(logs.CloseTarWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.CloseTarWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } if err := gzipWriter.Close(); err != nil { - log.Error(logs.CloseGzipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.CloseGzipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } }() var objectsWritten int buf := make([]byte, 3<<20) // the same as for upload - errIter := resSearch.Iterate(h.putObjectToArchive(ctx, log, bktInfo.CID, buf, + errIter := resSearch.Iterate(h.putObjectToArchive(ctx, bktInfo.CID, buf, func(obj *object.Object) (io.Writer, error) { objectsWritten++ return h.createTarFile(tarWriter, obj) }), ) if errIter != nil { - log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) } else if objectsWritten == 0 { - log.Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) } } } @@ -240,9 +238,9 @@ func (h *Handler) createTarFile(tw *tar.Writer, obj *object.Object) (io.Writer, }) } -func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID cid.ID, buf []byte, createArchiveHeader func(obj *object.Object) (io.Writer, error)) func(id oid.ID) bool { +func (h *Handler) putObjectToArchive(ctx context.Context, cnrID cid.ID, buf []byte, createArchiveHeader func(obj *object.Object) (io.Writer, error)) func(id oid.ID) bool { return func(id oid.ID) bool { - log = log.With(zap.String("oid", id.EncodeToString())) + logger := h.reqLogger(ctx).With(zap.String("oid", id.EncodeToString())) prm := PrmObjectGet{ PrmAuth: PrmAuth{ @@ -253,18 +251,18 @@ func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID resGet, err := h.frostfs.GetObject(ctx, prm) if err != nil { - log.Error(logs.FailedToGetObject, zap.Error(err), logs.TagField(logs.TagExternalStorage)) + logger.Error(logs.FailedToGetObject, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return false } fileWriter, err := createArchiveHeader(&resGet.Header) if err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) + logger.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) return false } if err = writeToArchive(resGet, fileWriter, buf); err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) + logger.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) return false } @@ -272,28 +270,17 @@ func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID } } -func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, cnrID cid.ID) (ResObjectSearch, error) { - scid, _ := c.UserValue("cid").(string) - prefix, _ := c.UserValue("prefix").(string) - - ctx := utils.GetContextFromRequest(c) - +func (h *Handler) searchObjectsByPrefix(ctx context.Context, cnrID cid.ID, prefix string) (ResObjectSearch, error) { prefix, err := url.QueryUnescape(prefix) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), - zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) - return nil, err + return nil, fmt.Errorf("unescape prefix: %w", err) } - log = log.With(zap.String("cid", scid), zap.String("prefix", prefix)) - resSearch, err := h.search(ctx, cnrID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage)) - ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) - return nil, err + return nil, fmt.Errorf("search objects by prefix: %w", err) } + return resSearch, nil } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 48f8f55..a982bc2 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -16,7 +16,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -144,6 +143,10 @@ var ( ErrGatewayTimeout = errors.New("gateway timeout") // ErrQuotaLimitReached is returned from FrostFS in case of quota exceeded. ErrQuotaLimitReached = errors.New("quota limit reached") + // ErrContainerNotFound is returned from FrostFS in case of container was not found. + ErrContainerNotFound = errors.New("container not found") + // ErrObjectNotFound is returned from FrostFS in case of object was not found. + ErrObjectNotFound = errors.New("object not found") ) // FrostFS represents virtual connection to FrostFS network. @@ -203,7 +206,7 @@ func New(params *AppParams, config Config, tree layer.TreeService, workerPool *a // byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID, objID oid.ID, handler func(context.Context, request, oid.Address)) { +func (h *Handler) byNativeAddress(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, objID oid.ID, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { ctx, span := tracing.StartSpanFromContext(ctx, "handler.byNativeAddress") defer span.End() @@ -213,72 +216,59 @@ func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID // byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that // resolves object address from S3-like path /. -func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path string, handler func(context.Context, request, oid.Address)) { +func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path") defer span.End() - c, log := req.RequestCtx, req.log - foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) if err != nil { - log.Error(logs.FailedToGetLatestVersionOfObject, zap.Error(err), zap.String("cid", cnrID.String()), - zap.String("path", path), logs.TagField(logs.TagExternalStorageTree)) - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetLatestVersionOfObject, err, zap.String("path", path)) return } if foundOID.IsDeleteMarker { - log.Error(logs.ObjectWasDeleted, logs.TagField(logs.TagExternalStorageTree)) - ResponseError(c, "object deleted", fasthttp.StatusNotFound) + h.logAndSendError(ctx, req, logs.ObjectWasDeleted, ErrObjectNotFound) return } addr := newAddress(cnrID, foundOID.OID) - handler(ctx, newRequest(c, log), addr) + handler(ctx, req, addr) } // byAttribute is a wrapper similar to byNativeAddress. -func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Context, request, oid.Address)) { - cidParam, _ := c.UserValue("cid").(string) - key, _ := c.UserValue("attr_key").(string) - val, _ := c.UserValue("attr_val").(string) - - ctx := utils.GetContextFromRequest(c) - log := utils.GetReqLogOrDefault(ctx, h.log) +func (h *Handler) byAttribute(ctx context.Context, req *fasthttp.RequestCtx, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { + cidParam, _ := req.UserValue("cid").(string) + key, _ := req.UserValue("attr_key").(string) + val, _ := req.UserValue("attr_val").(string) key, err := url.QueryUnescape(key) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), - zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToUnescapeQuery, err, zap.String("cid", cidParam), zap.String("attr_key", key)) return } val, err = url.QueryUnescape(val) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), - zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToUnescapeQuery, err, zap.String("cid", cidParam), zap.String("attr_val", key)) return } val = prepareAtribute(key, val) - log = log.With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val)) + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(zap.String("cid", cidParam), + zap.String("attr_key", key), zap.String("attr_val", val))) - bktInfo, err := h.getBucketInfo(ctx, cidParam, log) + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } - objID, err := h.findObjectByAttribute(ctx, log, bktInfo.CID, key, val) + objID, err := h.findObjectByAttribute(ctx, bktInfo.CID, key, val) if err != nil { if errors.Is(err, io.EOF) { - ResponseError(c, err.Error(), fasthttp.StatusNotFound) - return + err = fmt.Errorf("%w: %s", ErrObjectNotFound, err.Error()) } - - ResponseError(c, err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToFindObjectByAttribute, err) return } @@ -286,14 +276,13 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte addr.SetContainer(bktInfo.CID) addr.SetObject(objID) - handler(ctx, newRequest(c, log), addr) + handler(ctx, req, addr) } -func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { +func (h *Handler) findObjectByAttribute(ctx context.Context, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { res, err := h.search(ctx, cnrID, attrKey, attrVal, object.MatchStringEqual) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage)) - return oid.ID{}, fmt.Errorf("could not search for objects: %w", err) + return oid.ID{}, fmt.Errorf("search objects: %w", err) } defer res.Close() @@ -303,14 +292,14 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cn if n == 0 { switch { case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): - log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) + h.reqLogger(ctx).Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) attrVal = prepareAtribute(attrFileName, attrVal) - return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal) + return h.findObjectByAttribute(ctx, cnrID, attrFileName, attrVal) case errors.Is(err, io.EOF): - log.Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) + h.reqLogger(ctx).Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("object not found: %w", err) default: - log.Error(logs.ReadObjectListFailed, zap.Error(err), logs.TagField(logs.TagExternalStorage)) + h.reqLogger(ctx).Error(logs.ReadObjectListFailed, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("read object list failed: %w", err) } } @@ -369,13 +358,13 @@ func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*ci zone := h.config.FormContainerZone(namespace) cnrID, err = h.containerResolver.Resolve(ctx, zone, containerID) if err != nil && strings.Contains(err.Error(), "not found") { - err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error()) + err = fmt.Errorf("%w: %s", ErrContainerNotFound, err.Error()) } } return cnrID, err } -func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log *zap.Logger) (*data.BucketInfo, error) { +func (h *Handler) getBucketInfo(ctx context.Context, containerName string) (*data.BucketInfo, error) { ns, err := middleware.GetNamespace(ctx) if err != nil { return nil, err @@ -387,21 +376,16 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log * cnrID, err := h.resolveContainer(ctx, containerName) if err != nil { - log.Error(logs.CouldNotResolveContainerID, zap.Error(err), zap.String("cnrName", containerName), - logs.TagField(logs.TagDatapath)) - return nil, err + return nil, fmt.Errorf("resolve container: %w", err) } bktInfo, err := h.readContainer(ctx, *cnrID) if err != nil { - log.Error(logs.CouldNotGetContainerInfo, zap.Error(err), zap.String("cnrName", containerName), - zap.String("cnrName", cnrID.String()), - logs.TagField(logs.TagExternalStorage)) - return nil, err + return nil, fmt.Errorf("read container: %w", err) } if err = h.cache.Put(bktInfo); err != nil { - log.Warn(logs.CouldntPutBucketIntoCache, + h.reqLogger(ctx).Warn(logs.CouldntPutBucketIntoCache, zap.String("bucket name", bktInfo.Name), zap.Stringer("bucket cid", bktInfo.CID), zap.Error(err), @@ -434,31 +418,24 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } -func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.browseIndex") +func (h *Handler) browseIndex(ctx context.Context, req *fasthttp.RequestCtx, cidParam, oidParam string, isNativeList bool) { + ctx, span := tracing.StartSpanFromContext(ctx, "handler.browseIndex") defer span.End() - utils.SetContextToRequest(ctx, c) if !h.config.IndexPageEnabled() { - c.SetStatusCode(fasthttp.StatusNotFound) + req.SetStatusCode(fasthttp.StatusNotFound) return } - cidURLParam := c.UserValue("cid").(string) - oidURLParam := c.UserValue("oid").(string) - - reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With(zap.String("cid", cidURLParam), zap.String("oid", oidURLParam)) - - unescapedKey, err := url.QueryUnescape(oidURLParam) + unescapedKey, err := url.QueryUnescape(oidParam) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToUnescapeOIDParam, err) return } - bktInfo, err := h.getBucketInfo(ctx, cidURLParam, log) + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } @@ -468,7 +445,7 @@ func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) { listFunc = h.getDirObjectsNative } - h.browseObjects(c, browseParams{ + h.browseObjects(ctx, req, browseParams{ bucketInfo: bktInfo, prefix: unescapedKey, listObjects: listFunc, diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 3a81c50..93cb1d9 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -409,7 +409,7 @@ func TestFindObjectByAttribute(t *testing.T) { obj.SetAttributes(tc.firstAttr, tc.secondAttr) hc.cfg.additionalSearch = tc.additionalSearch - objID, err := hc.Handler().findObjectByAttribute(ctx, hc.Handler().log, cnrID, tc.reqAttrKey, tc.reqAttrValue) + objID, err := hc.Handler().findObjectByAttribute(ctx, cnrID, tc.reqAttrKey, tc.reqAttrValue) if tc.err != "" { require.Error(t, err) require.Contains(t, err.Error(), tc.err) diff --git a/internal/handler/head.go b/internal/handler/head.go index 7718c9c..11d45fc 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -27,7 +27,7 @@ const ( hdrContainerID = "X-Container-Id" ) -func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid.Address) { +func (h *Handler) headObject(ctx context.Context, req *fasthttp.RequestCtx, objectAddress oid.Address) { var start = time.Now() btoken := bearerToken(ctx) @@ -41,7 +41,7 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid obj, err := h.frostfs.HeadObject(ctx, prm) if err != nil { - req.handleFrostFSErr(err, start) + h.logAndSendError(ctx, req, logs.FailedToHeadObject, err, zap.Stringer("elapsed", time.Since(start))) return } @@ -65,7 +65,7 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid case object.AttributeTimestamp: value, err := strconv.ParseInt(val, 10, 64) if err != nil { - req.log.Info(logs.CouldntParseCreationDate, + h.reqLogger(ctx).Info(logs.CouldntParseCreationDate, zap.String("key", key), zap.String("val", val), zap.Error(err), @@ -100,7 +100,7 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid return h.frostfs.RangeObject(ctx, prmRange) }, filename) if err != nil && err != io.EOF { - req.handleFrostFSErr(err, start) + h.logAndSendError(ctx, req, logs.FailedToDetectContentTypeFromPayload, err, zap.Stringer("elapsed", time.Since(start))) return } } @@ -116,48 +116,44 @@ func idsToResponse(resp *fasthttp.Response, obj *object.Object) { } // HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format. -func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.HeadByAddressOrBucketName") +func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.HeadByAddressOrBucketName") defer span.End() - cidParam, _ := c.UserValue("cid").(string) - oidParam, _ := c.UserValue("oid").(string) + cidParam, _ := req.UserValue("cid").(string) + oidParam, _ := req.UserValue("oid").(string) - log := utils.GetReqLogOrDefault(ctx, h.log).With( + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( zap.String("cid", cidParam), zap.String("oid", oidParam), - ) + )) - bktInfo, err := h.getBucketInfo(ctx, cidParam, log) + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } + checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { - log.Error(logs.FailedToCheckIfSettingsNodeExist, zap.String("cid", bktInfo.CID.String()), - zap.Error(checkS3Err), logs.TagField(logs.TagExternalStorageTree)) - logAndSendBucketError(c, log, checkS3Err) + h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) return } - req := newRequest(c, log) - var objID oid.ID if checkS3Err == nil { h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject) } else if err = objID.DecodeString(oidParam); err == nil { h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) } else { - logAndSendBucketError(c, log, checkS3Err) + h.logAndSendError(ctx, req, logs.InvalidOIDParam, err) } } // HeadByAttribute handles attribute-based head requests. -func (h *Handler) HeadByAttribute(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.HeadByAttribute") +func (h *Handler) HeadByAttribute(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.HeadByAttribute") defer span.End() - utils.SetContextToRequest(ctx, c) - h.byAttribute(c, h.headObject) + h.byAttribute(ctx, req, h.headObject) } diff --git a/internal/handler/multipart.go b/internal/handler/multipart.go index 5ed2350..5b06882 100644 --- a/internal/handler/multipart.go +++ b/internal/handler/multipart.go @@ -1,6 +1,7 @@ package handler import ( + "context" "errors" "io" "strconv" @@ -53,7 +54,7 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF } // getPayload returns initial payload if object is not multipart else composes new reader with parts data. -func (h *Handler) getPayload(p getMultiobjectBodyParams) (io.ReadCloser, uint64, error) { +func (h *Handler) getPayload(ctx context.Context, p getMultiobjectBodyParams) (io.ReadCloser, uint64, error) { cid, ok := p.obj.Header.ContainerID() if !ok { return nil, 0, errors.New("no container id set") @@ -66,7 +67,6 @@ func (h *Handler) getPayload(p getMultiobjectBodyParams) (io.ReadCloser, uint64, if err != nil { return nil, 0, err } - ctx := p.req.RequestCtx params := PrmInitMultiObjectReader{ Addr: newAddress(cid, oid), Bearer: bearerToken(ctx), diff --git a/internal/handler/reader.go b/internal/handler/reader.go index e8ac098..711bfd2 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -63,11 +63,10 @@ func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error), file type getMultiobjectBodyParams struct { obj *Object - req request strSize string } -func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.Address) { +func (h *Handler) receiveFile(ctx context.Context, req *fasthttp.RequestCtx, objAddress oid.Address) { var ( shouldDownload = req.QueryArgs().GetBool("download") start = time.Now() @@ -85,12 +84,12 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A rObj, err := h.frostfs.GetObject(ctx, prm) if err != nil { - req.handleFrostFSErr(err, start) + h.logAndSendError(ctx, req, logs.FailedToGetObject, err, zap.Stringer("elapsed", time.Since(start))) return } // we can't close reader in this function, so how to do it? - req.setIDs(rObj.Header) + setIDs(req, rObj.Header) payload := rObj.Payload payloadSize := rObj.Header.PayloadSize() for _, attr := range rObj.Header.Attributes() { @@ -107,8 +106,8 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A case object.AttributeFileName: filename = val case object.AttributeTimestamp: - if err = req.setTimestamp(val); err != nil { - req.log.Error(logs.CouldntParseCreationDate, + if err = setTimestamp(req, val); err != nil { + h.reqLogger(ctx).Error(logs.CouldntParseCreationDate, zap.String("val", val), zap.Error(err), logs.TagField(logs.TagDatapath)) @@ -118,13 +117,12 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A case object.AttributeFilePath: filepath = val case attributeMultipartObjectSize: - payload, payloadSize, err = h.getPayload(getMultiobjectBodyParams{ + payload, payloadSize, err = h.getPayload(ctx, getMultiobjectBodyParams{ obj: rObj, - req: req, strSize: val, }) if err != nil { - req.handleFrostFSErr(err, start) + h.logAndSendError(ctx, req, logs.FailedToGetObjectPayload, err, zap.Stringer("elapsed", time.Since(start))) return } } @@ -133,7 +131,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A filename = filepath } - req.setDisposition(shouldDownload, filename) + setDisposition(req, shouldDownload, filename) req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10)) @@ -145,8 +143,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A return payload, nil }, filename) if err != nil && err != io.EOF { - req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToDetectContentTypeFromPayload, err, zap.Stringer("elapsed", time.Since(start))) return } @@ -165,7 +162,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A req.Response.SetBodyStream(payload, int(payloadSize)) } -func (r *request) setIDs(obj object.Object) { +func setIDs(r *fasthttp.RequestCtx, obj object.Object) { objID, _ := obj.ID() cnrID, _ := obj.ContainerID() r.Response.Header.Set(hdrObjectID, objID.String()) @@ -173,7 +170,7 @@ func (r *request) setIDs(obj object.Object) { r.Response.Header.Set(hdrContainerID, cnrID.String()) } -func (r *request) setDisposition(shouldDownload bool, filename string) { +func setDisposition(r *fasthttp.RequestCtx, shouldDownload bool, filename string) { const ( inlineDisposition = "inline" attachmentDisposition = "attachment" @@ -187,7 +184,7 @@ func (r *request) setDisposition(shouldDownload bool, filename string) { r.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename)) } -func (r *request) setTimestamp(timestamp string) error { +func setTimestamp(r *fasthttp.RequestCtx, timestamp string) error { value, err := strconv.ParseInt(timestamp, 10, 64) if err != nil { return err diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 48d0495..05f4c97 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -50,44 +50,41 @@ func (pr *putResponse) encode(w io.Writer) error { } // Upload handles multipart upload request. -func (h *Handler) Upload(c *fasthttp.RequestCtx) { - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.Upload") +func (h *Handler) Upload(req *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(req), "handler.Upload") defer span.End() - utils.SetContextToRequest(ctx, c) var file MultipartFile - scid, _ := c.UserValue("cid").(string) - bodyStream := c.RequestBodyStream() + scid, _ := req.UserValue("cid").(string) + bodyStream := req.RequestBodyStream() drainBuf := make([]byte, drainBufSize) - reqLog := utils.GetReqLogOrDefault(ctx, h.log) - log := reqLog.With(zap.String("cid", scid)) + log := h.reqLogger(ctx) + ctx = utils.SetReqLog(ctx, log.With(zap.String("cid", scid))) - bktInfo, err := h.getBucketInfo(ctx, scid, log) + bktInfo, err := h.getBucketInfo(ctx, scid) if err != nil { - logAndSendBucketError(c, log, err) + h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) return } - boundary := string(c.Request.Header.MultipartFormBoundary()) + boundary := string(req.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { - log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.CouldNotReceiveMultipartForm, err) return } - filtered, err := filterHeaders(log, &c.Request.Header) + filtered, err := filterHeaders(log, &req.Request.Header) if err != nil { - log.Error(logs.FailedToFilterHeaders, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToFilterHeaders, err) return } - if c.Request.Header.Peek(explodeArchiveHeader) != nil { - h.explodeArchive(request{c, log}, bktInfo, file, filtered) + if req.Request.Header.Peek(explodeArchiveHeader) != nil { + h.explodeArchive(ctx, req, bktInfo, file, filtered) } else { - h.uploadSingleObject(request{c, log}, bktInfo, file, filtered) + h.uploadSingleObject(ctx, req, bktInfo, file, filtered) } // Multipart is multipart and thus can contain more than one part which @@ -104,46 +101,39 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { } } -func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file MultipartFile, filtered map[string]string) { - c, log := req.RequestCtx, req.log - - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.uploadSingleObject") +func (h *Handler) uploadSingleObject(ctx context.Context, req *fasthttp.RequestCtx, bkt *data.BucketInfo, file MultipartFile, filtered map[string]string) { + ctx, span := tracing.StartSpanFromContext(ctx, "handler.uploadSingleObject") defer span.End() - utils.SetContextToRequest(ctx, c) setIfNotExist(filtered, object.AttributeFileName, file.FileName()) - attributes, err := h.extractAttributes(c, log, filtered) + attributes, err := h.extractAttributes(ctx, req, filtered) if err != nil { - log.Error(logs.FailedToGetAttributes, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToGetAttributes, err) return } - idObj, err := h.uploadObject(c, bkt, attributes, file) + idObj, err := h.uploadObject(ctx, bkt, attributes, file) if err != nil { - h.handlePutFrostFSErr(c, err, log) + h.logAndSendError(ctx, req, logs.FailedToUploadObject, err) return } - log.Debug(logs.ObjectUploaded, + h.reqLogger(ctx).Debug(logs.ObjectUploaded, zap.String("oid", idObj.EncodeToString()), zap.String("FileName", file.FileName()), logs.TagField(logs.TagExternalStorage), ) addr := newAddress(bkt.CID, idObj) - c.Response.Header.SetContentType(jsonHeader) + req.Response.Header.SetContentType(jsonHeader) // Try to return the response, otherwise, if something went wrong, throw an error. - if err = newPutResponse(addr).encode(c); err != nil { - log.Error(logs.CouldNotEncodeResponse, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) + if err = newPutResponse(addr).encode(req); err != nil { + h.logAndSendError(ctx, req, logs.CouldNotEncodeResponse, err) return } } -func (h *Handler) uploadObject(c *fasthttp.RequestCtx, bkt *data.BucketInfo, attrs []object.Attribute, file io.Reader) (oid.ID, error) { - ctx := utils.GetContextFromRequest(c) - +func (h *Handler) uploadObject(ctx context.Context, bkt *data.BucketInfo, attrs []object.Attribute, file io.Reader) (oid.ID, error) { obj := object.New() obj.SetContainerID(bkt.CID) obj.SetOwnerID(*h.ownerID) @@ -168,19 +158,18 @@ func (h *Handler) uploadObject(c *fasthttp.RequestCtx, bkt *data.BucketInfo, att return idObj, nil } -func (h *Handler) extractAttributes(c *fasthttp.RequestCtx, log *zap.Logger, filtered map[string]string) ([]object.Attribute, error) { - ctx := utils.GetContextFromRequest(c) +func (h *Handler) extractAttributes(ctx context.Context, req *fasthttp.RequestCtx, filtered map[string]string) ([]object.Attribute, error) { now := time.Now() - if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { + if rawHeader := req.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { - log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err), + h.reqLogger(ctx).Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err), logs.TagField(logs.TagDatapath)) } else { now = parsed } } if err := utils.PrepareExpirationHeader(ctx, h.frostfs, filtered, now); err != nil { - log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err), logs.TagField(logs.TagDatapath)) return nil, err } attributes := make([]object.Attribute, 0, len(filtered)) @@ -207,38 +196,33 @@ func newAttribute(key string, val string) object.Attribute { // explodeArchive read files from archive and creates objects for each of them. // Sets FilePath attribute with name from tar.Header. -func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.ReadCloser, filtered map[string]string) { - c, log := req.RequestCtx, req.log - - ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.explodeArchive") +func (h *Handler) explodeArchive(ctx context.Context, req *fasthttp.RequestCtx, bkt *data.BucketInfo, file io.ReadCloser, filtered map[string]string) { + ctx, span := tracing.StartSpanFromContext(ctx, "handler.explodeArchive") defer span.End() - utils.SetContextToRequest(ctx, c) // remove user attributes which vary for each file in archive // to guarantee that they won't appear twice delete(filtered, object.AttributeFileName) delete(filtered, object.AttributeFilePath) - commonAttributes, err := h.extractAttributes(c, log, filtered) + commonAttributes, err := h.extractAttributes(ctx, req, filtered) if err != nil { - log.Error(logs.FailedToGetAttributes, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToGetAttributes, err) return } attributes := commonAttributes reader := file - if bytes.EqualFold(c.Request.Header.Peek(fasthttp.HeaderContentEncoding), []byte("gzip")) { - log.Debug(logs.GzipReaderSelected, logs.TagField(logs.TagDatapath)) + if bytes.EqualFold(req.Request.Header.Peek(fasthttp.HeaderContentEncoding), []byte("gzip")) { + h.reqLogger(ctx).Debug(logs.GzipReaderSelected, logs.TagField(logs.TagDatapath)) gzipReader, err := gzip.NewReader(file) if err != nil { - log.Error(logs.FailedToCreateGzipReader, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could read gzip file: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToCreateGzipReader, err) return } defer func() { if err := gzipReader.Close(); err != nil { - log.Warn(logs.FailedToCloseReader, zap.Error(err), logs.TagField(logs.TagDatapath)) + h.reqLogger(ctx).Warn(logs.FailedToCloseReader, zap.Error(err), logs.TagField(logs.TagDatapath)) } }() reader = gzipReader @@ -250,8 +234,7 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read if errors.Is(err, io.EOF) { break } else if err != nil { - log.Error(logs.FailedToReadFileFromTar, zap.Error(err), logs.TagField(logs.TagDatapath)) - ResponseError(c, "could not get next entry: "+err.Error(), fasthttp.StatusBadRequest) + h.logAndSendError(ctx, req, logs.FailedToReadFileFromTar, err) return } @@ -265,13 +248,13 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read attributes = append(attributes, newAttribute(object.AttributeFilePath, obj.Name)) attributes = append(attributes, newAttribute(object.AttributeFileName, fileName)) - idObj, err := h.uploadObject(c, bkt, attributes, tarReader) + idObj, err := h.uploadObject(ctx, bkt, attributes, tarReader) if err != nil { - h.handlePutFrostFSErr(c, err, log) + h.logAndSendError(ctx, req, logs.FailedToUploadObject, err) return } - log.Debug(logs.ObjectUploaded, + h.reqLogger(ctx).Debug(logs.ObjectUploaded, zap.String("oid", idObj.EncodeToString()), zap.String("FileName", fileName), logs.TagField(logs.TagExternalStorage), @@ -279,14 +262,6 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read } } -func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *zap.Logger) { - statusCode, msg, additionalFields := formErrorResponse("could not store file in frostfs", err) - logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) - - log.Error(logs.CouldNotStoreFileInFrostfs, append(logFields, logs.TagField(logs.TagExternalStorage))...) - ResponseError(r, msg, statusCode) -} - func (h *Handler) fetchBearerToken(ctx context.Context) *bearer.Token { if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil { return tkn diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 0a1dc62..8cb070d 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -5,13 +5,12 @@ import ( "errors" "fmt" "strings" - "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -19,30 +18,6 @@ import ( "go.uber.org/zap" ) -type request struct { - *fasthttp.RequestCtx - log *zap.Logger -} - -func newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) request { - return request{ - RequestCtx: ctx, - log: log, - } -} - -func (r *request) handleFrostFSErr(err error, start time.Time) { - logFields := []zap.Field{ - zap.Stringer("elapsed", time.Since(start)), - zap.Error(err), - } - statusCode, msg, additionalFields := formErrorResponse("could not receive object", err) - logFields = append(logFields, additionalFields...) - - r.log.Error(logs.CouldNotReceiveObject, append(logFields, logs.TagField(logs.TagExternalStorage))...) - ResponseError(r.RequestCtx, msg, statusCode) -} - func bearerToken(ctx context.Context) *bearer.Token { if tkn, err := tokens.LoadBearerToken(ctx); err == nil { return tkn @@ -84,14 +59,16 @@ func isValidValue(s string) bool { return true } -func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { - log.Error(logs.CouldNotGetBucket, zap.Error(err), logs.TagField(logs.TagDatapath)) +func (h *Handler) reqLogger(ctx context.Context) *zap.Logger { + return utils.GetReqLogOrDefault(ctx, h.log) +} - if client.IsErrContainerNotFound(err) { - ResponseError(c, "Not Found", fasthttp.StatusNotFound) - return - } - ResponseError(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) +func (h *Handler) logAndSendError(ctx context.Context, c *fasthttp.RequestCtx, msg string, err error, additional ...zap.Field) { + utils.GetReqLogOrDefault(ctx, h.log).Error(msg, + append([]zap.Field{zap.Error(err), logs.TagField(logs.TagDatapath)}, additional...)...) + + msg, code := formErrorResponse(err) + ResponseError(c, msg, code) } func newAddress(cnr cid.ID, obj oid.ID) oid.Address { @@ -112,31 +89,23 @@ func ResponseError(r *fasthttp.RequestCtx, msg string, code int) { r.Error(msg+"\n", code) } -func formErrorResponse(message string, err error) (int, string, []zap.Field) { - var ( - msg string - statusCode int - logFields []zap.Field - ) - - st := new(sdkstatus.ObjectAccessDenied) - +func formErrorResponse(err error) (string, int) { switch { - case errors.As(err, &st): - statusCode = fasthttp.StatusForbidden - reason := st.Reason() - msg = fmt.Sprintf("%s: %v: %s", message, err, reason) - logFields = append(logFields, zap.String("error_detail", reason)) + case errors.Is(err, ErrAccessDenied): + return fmt.Sprintf("Storage Access Denied:\n%v", err), fasthttp.StatusForbidden + case errors.Is(err, layer.ErrNodeAccessDenied): + return fmt.Sprintf("Tree Access Denied:\n%v", err), fasthttp.StatusForbidden case errors.Is(err, ErrQuotaLimitReached): - statusCode = fasthttp.StatusConflict - msg = fmt.Sprintf("%s: %v", message, err) - case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err): - statusCode = fasthttp.StatusNotFound - msg = "Not Found" + return fmt.Sprintf("Quota Reached:\n%v", err), fasthttp.StatusConflict + case errors.Is(err, ErrContainerNotFound): + return fmt.Sprintf("Container Not Found:\n%v", err), fasthttp.StatusNotFound + case errors.Is(err, ErrObjectNotFound): + return fmt.Sprintf("Object Not Found:\n%v", err), fasthttp.StatusNotFound + case errors.Is(err, layer.ErrNodeNotFound): + return fmt.Sprintf("Tree Node Not Found:\n%v", err), fasthttp.StatusNotFound + case errors.Is(err, ErrGatewayTimeout): + return fmt.Sprintf("Gateway Timeout:\n%v", err), fasthttp.StatusGatewayTimeout default: - statusCode = fasthttp.StatusBadRequest - msg = fmt.Sprintf("%s: %v", message, err) + return fmt.Sprintf("Bad Request:\n%v", err), fasthttp.StatusBadRequest } - - return statusCode, msg, logFields } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 3166f98..3e9b931 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -78,7 +78,7 @@ const ( // Log messages with the "datapath" tag. const ( CouldntParseCreationDate = "couldn't parse creation date" - CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" + FailedToDetectContentTypeFromPayload = "failed to detect Content-Type from payload" FailedToAddObjectToArchive = "failed to add object to archive" CloseZipWriter = "close zip writer" IgnorePartEmptyFormName = "ignore part, empty form name" @@ -105,9 +105,21 @@ const ( CouldNotReceiveMultipartForm = "could not receive multipart/form" ObjectsNotFound = "objects not found" IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" - CouldNotGetBucket = "could not get bucket" - CouldNotResolveContainerID = "could not resolve container id" - FailedToSumbitTaskToPool = "failed to submit task to pool" + FailedToGetBucketInfo = "could not get bucket info" + FailedToSubmitTaskToPool = "failed to submit task to pool" + ObjectWasDeleted = "object was deleted" + FailedToGetLatestVersionOfObject = "failed to get latest version of object" + FailedToCheckIfSettingsNodeExist = "failed to check if settings node exists" + FailedToListObjects = "failed to list objects" + FailedToParseTemplate = "failed to parse template" + FailedToExecuteTemplate = "failed to execute template" + FailedToUploadObject = "failed to upload object" + FailedToHeadObject = "failed to head object" + FailedToGetObject = "failed to get object" + FailedToGetObjectPayload = "failed to get object payload" + FailedToFindObjectByAttribute = "failed to get find object by attribute" + FailedToUnescapeOIDParam = "failed to unescape oid param" + InvalidOIDParam = "invalid oid param" CouldNotGetCORSConfiguration = "could not get cors configuration" EmptyOriginRequestHeader = "empty Origin request header" EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header" @@ -117,21 +129,13 @@ const ( // Log messages with the "external_storage" tag. const ( - CouldNotReceiveObject = "could not receive object" - CouldNotSearchForObjects = "could not search for objects" ObjectNotFound = "object not found" ReadObjectListFailed = "read object list failed" - CouldNotStoreFileInFrostfs = "could not store file in frostfs" - FailedToHeadObject = "failed to head object" ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" - FailedToGetObject = "failed to get object" ObjectUploaded = "object uploaded" - CouldNotGetContainerInfo = "could not get container info" ) // Log messages with the "external_storage_tree" tag. const ( - ObjectWasDeleted = "object was deleted" - FailedToGetLatestVersionOfObject = "failed to get latest version of object" - FailedToCheckIfSettingsNodeExist = "Failed to check if settings node exists" + FoundSeveralSystemTreeNodes = "found several system tree nodes" ) diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index 4cf45a4..1841446 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -45,7 +46,7 @@ func (x *FrostFS) Container(ctx context.Context, containerPrm handler.PrmContain res, err := x.pool.GetContainer(ctx, prm) if err != nil { - return nil, handleObjectError("read container via connection pool", err) + return nil, handleStorageError("read container via connection pool", err) } return &res, nil @@ -69,7 +70,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) idObj, err := x.pool.PutObject(ctx, prmPut) if err != nil { - return oid.ID{}, handleObjectError("save object via connection pool", err) + return oid.ID{}, handleStorageError("save object via connection pool", err) } return idObj.ObjectID, nil } @@ -85,7 +86,7 @@ func (x payloadReader) Read(p []byte) (int, error) { if err != nil && errors.Is(err, io.EOF) { return n, err } - return n, handleObjectError("read payload", err) + return n, handleStorageError("read payload", err) } // HeadObject implements frostfs.FrostFS interface method. @@ -102,7 +103,7 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o res, err := x.pool.HeadObject(ctx, prmHead) if err != nil { - return nil, handleObjectError("read object header via connection pool", err) + return nil, handleStorageError("read object header via connection pool", err) } return &res, nil @@ -122,7 +123,7 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han res, err := x.pool.GetObject(ctx, prmGet) if err != nil { - return nil, handleObjectError("init full object reading via connection pool", err) + return nil, handleStorageError("init full object reading via connection pool", err) } return &handler.Object{ @@ -147,7 +148,7 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) ( res, err := x.pool.ObjectRange(ctx, prmRange) if err != nil { - return nil, handleObjectError("init payload range reading via connection pool", err) + return nil, handleStorageError("init payload range reading via connection pool", err) } return payloadReader{&res}, nil @@ -168,7 +169,7 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch res, err := x.pool.SearchObjects(ctx, prmSearch) if err != nil { - return nil, handleObjectError("init object search via connection pool", err) + return nil, handleStorageError("init object search via connection pool", err) } return &res, nil @@ -202,7 +203,7 @@ func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) { netmapSnapshot, err := x.pool.NetMapSnapshot(ctx) if err != nil { - return netmapSnapshot, handleObjectError("get netmap via connection pool", err) + return netmapSnapshot, handleStorageError("get netmap via connection pool", err) } return netmapSnapshot, nil @@ -226,7 +227,7 @@ func (x *ResolverFrostFS) SystemDNS(ctx context.Context) (string, error) { networkInfo, err := x.pool.NetworkInfo(ctx) if err != nil { - return "", handleObjectError("read network info via client", err) + return "", handleStorageError("read network info via client", err) } domain := networkInfo.RawNetworkParameter("SystemDNS") @@ -237,7 +238,7 @@ func (x *ResolverFrostFS) SystemDNS(ctx context.Context) (string, error) { return string(domain), nil } -func handleObjectError(msg string, err error) error { +func handleStorageError(msg string, err error) error { if err == nil { return nil } @@ -250,6 +251,14 @@ func handleObjectError(msg string, err error) error { return fmt.Errorf("%s: %w: %s", msg, handler.ErrAccessDenied, reason) } + if client.IsErrContainerNotFound(err) { + return fmt.Errorf("%s: %w: %s", msg, handler.ErrContainerNotFound, err.Error()) + } + + if client.IsErrObjectNotFound(err) { + return fmt.Errorf("%s: %w: %s", msg, handler.ErrObjectNotFound, err.Error()) + } + if IsTimeoutError(err) { return fmt.Errorf("%s: %w: %s", msg, handler.ErrGatewayTimeout, err.Error()) } diff --git a/internal/service/frostfs/frostfs_test.go b/internal/service/frostfs/frostfs_test.go index e9b3329..e4344f7 100644 --- a/internal/service/frostfs/frostfs_test.go +++ b/internal/service/frostfs/frostfs_test.go @@ -18,7 +18,7 @@ func TestHandleObjectError(t *testing.T) { msg := "some msg" t.Run("nil error", func(t *testing.T) { - err := handleObjectError(msg, nil) + err := handleStorageError(msg, nil) require.Nil(t, err) }) @@ -27,7 +27,7 @@ func TestHandleObjectError(t *testing.T) { inputErr := new(apistatus.ObjectAccessDenied) inputErr.WriteReason(reason) - err := handleObjectError(msg, inputErr) + err := handleStorageError(msg, inputErr) require.ErrorIs(t, err, handler.ErrAccessDenied) require.Contains(t, err.Error(), reason) require.Contains(t, err.Error(), msg) @@ -38,7 +38,7 @@ func TestHandleObjectError(t *testing.T) { inputErr := new(apistatus.ObjectAccessDenied) inputErr.WriteReason(reason) - err := handleObjectError(msg, inputErr) + err := handleStorageError(msg, inputErr) require.ErrorIs(t, err, handler.ErrQuotaLimitReached) require.Contains(t, err.Error(), reason) require.Contains(t, err.Error(), msg) @@ -47,7 +47,7 @@ func TestHandleObjectError(t *testing.T) { t.Run("simple timeout", func(t *testing.T) { inputErr := errors.New("timeout") - err := handleObjectError(msg, inputErr) + err := handleStorageError(msg, inputErr) require.ErrorIs(t, err, handler.ErrGatewayTimeout) require.Contains(t, err.Error(), inputErr.Error()) require.Contains(t, err.Error(), msg) @@ -58,7 +58,7 @@ func TestHandleObjectError(t *testing.T) { defer cancel() <-ctx.Done() - err := handleObjectError(msg, ctx.Err()) + err := handleStorageError(msg, ctx.Err()) require.ErrorIs(t, err, handler.ErrGatewayTimeout) require.Contains(t, err.Error(), ctx.Err().Error()) require.Contains(t, err.Error(), msg) @@ -67,7 +67,7 @@ func TestHandleObjectError(t *testing.T) { t.Run("grpc deadline exceeded", func(t *testing.T) { inputErr := fmt.Errorf("wrap grpc error: %w", status.Error(codes.DeadlineExceeded, "error")) - err := handleObjectError(msg, inputErr) + err := handleStorageError(msg, inputErr) require.ErrorIs(t, err, handler.ErrGatewayTimeout) require.Contains(t, err.Error(), inputErr.Error()) require.Contains(t, err.Error(), msg) @@ -76,7 +76,7 @@ func TestHandleObjectError(t *testing.T) { t.Run("unknown error", func(t *testing.T) { inputErr := errors.New("unknown error") - err := handleObjectError(msg, inputErr) + err := handleStorageError(msg, inputErr) require.ErrorIs(t, err, inputErr) require.Contains(t, err.Error(), msg) }) diff --git a/internal/service/frostfs/tree_pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go index 410acda..d0b5501 100644 --- a/internal/service/frostfs/tree_pool_wrapper.go +++ b/internal/service/frostfs/tree_pool_wrapper.go @@ -63,7 +63,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([ nodes, err := w.p.GetNodes(ctx, poolPrm) if err != nil { - return nil, handleError(err) + return nil, handleTreeError(err) } res := make([]tree.NodeResponse, len(nodes)) @@ -82,7 +82,7 @@ func getBearer(ctx context.Context) []byte { return token.Marshal() } -func handleError(err error) error { +func handleTreeError(err error) error { if err == nil { return nil } @@ -122,7 +122,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) if err != nil { - return nil, handleError(err) + return nil, handleTreeError(err) } var subtree []tree.NodeResponse @@ -133,7 +133,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, node, err = subTreeReader.Next() } if err != io.EOF { - return nil, handleError(err) + return nil, handleTreeError(err) } return subtree, nil diff --git a/tree/tree.go b/tree/tree.go index 315e5ad..2ee9356 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -8,14 +8,18 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" ) type ( Tree struct { service ServiceClient + log *zap.Logger } // ServiceClient is a client to interact with tree service. @@ -73,8 +77,8 @@ const ( ) // NewTree creates instance of Tree using provided address and create grpc connection. -func NewTree(service ServiceClient) *Tree { - return &Tree{service: service} +func NewTree(service ServiceClient, log *zap.Logger) *Tree { + return &Tree{service: service, log: log} } type Meta interface { @@ -257,6 +261,9 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name if len(nodes) == 0 { return nil, layer.ErrNodeNotFound } + if len(nodes) != 1 { + c.reqLogger(ctx).Warn(logs.FoundSeveralSystemTreeNodes, zap.String("name", name), logs.TagField(logs.TagExternalStorageTree)) + } return newMultiNode(nodes) } @@ -296,7 +303,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { } if targetIndexNode == -1 { - return nil, layer.ErrNodeNotFound + return nil, fmt.Errorf("latest version: %w", layer.ErrNodeNotFound) } return nodes[targetIndexNode], nil @@ -423,6 +430,10 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr return intermediateNodes, nil } +func (c *Tree) reqLogger(ctx context.Context) *zap.Logger { + return utils.GetReqLogOrDefault(ctx, c.log) +} + func GetFilename(node NodeResponse) string { for _, kv := range node.GetMeta() { if kv.GetKey() == FileNameKey { From f0b86c8ba7e861c8a482cf60ba32b837dee18efb Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 4 Mar 2025 14:22:10 +0300 Subject: [PATCH 537/548] [#191] Update integration tests Signed-off-by: Denis Kirillov --- cmd/http-gw/integration_test.go | 119 ++++++++++++++++++++++++++++---- 1 file changed, 104 insertions(+), 15 deletions(-) diff --git a/cmd/http-gw/integration_test.go b/cmd/http-gw/integration_test.go index 20b4c8b..6ab8e99 100644 --- a/cmd/http-gw/integration_test.go +++ b/cmd/http-gw/integration_test.go @@ -20,9 +20,11 @@ import ( containerv2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -99,6 +101,7 @@ func TestIntegration(t *testing.T) { t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID) }) t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, ownerID, CID) }) t.Run("test namespaces "+version, func(t *testing.T) { checkNamespaces(ctx, t, clientPool, ownerID, CID) }) + t.Run("test status codes "+version, func(t *testing.T) { checkStatusCodes(ctx, t, clientPool, ownerID, version) }) cancel() server.Wait() @@ -267,7 +270,7 @@ func putWithDuplicateKeys(t *testing.T, CID cid.ID) { body, err := io.ReadAll(resp.Body) require.NoError(t, err) - require.Equal(t, "key duplication error: "+attr+"\n", string(body)) + require.Contains(t, string(body), "key duplication error: "+attr+"\n") require.Equal(t, http.StatusBadRequest, resp.StatusCode) } @@ -436,7 +439,80 @@ func checkNamespaces(ctx context.Context, t *testing.T, clientPool *pool.Pool, o resp, err = http.DefaultClient.Do(req) require.NoError(t, err) require.Equal(t, http.StatusNotFound, resp.StatusCode) +} +func checkStatusCodes(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, version string) { + cli := http.Client{Timeout: 30 * time.Second} + + t.Run("container not found by name", func(t *testing.T) { + resp, err := cli.Get(testHost + "/get/unknown/object") + require.NoError(t, err) + require.Equal(t, http.StatusNotFound, resp.StatusCode) + requireBodyContains(t, resp, "container not found") + }) + + t.Run("container not found by cid", func(t *testing.T) { + cnrIDTest := cidtest.ID() + resp, err := cli.Get(testHost + "/get/" + cnrIDTest.EncodeToString() + "/object") + require.NoError(t, err) + requireBodyContains(t, resp, "container not found") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + + t.Run("object not found in storage", func(t *testing.T) { + resp, err := cli.Get(testHost + "/get_by_attribute/" + testContainerName + "/FilePath/object2") + require.NoError(t, err) + requireBodyContains(t, resp, "object not found") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + + t.Run("access denied", func(t *testing.T) { + basicACL := acl.Private + var recs []*eacl.Record + if version == "1.2.7" { + basicACL = acl.PublicRWExtended + rec := eacl.NewRecord() + rec.SetAction(eacl.ActionDeny) + rec.SetOperation(eacl.OperationGet) + recs = append(recs, rec) + } + + cnrID, err := createContainerBase(ctx, t, clientPool, ownerID, basicACL, "") + require.NoError(t, err) + + key, err := keys.NewPrivateKey() + require.NoError(t, err) + jsonToken, _ := makeBearerTokens(t, key, ownerID, version, recs...) + + t.Run("get", func(t *testing.T) { + request, err := http.NewRequest(http.MethodGet, testHost+"/get/"+cnrID.EncodeToString()+"/object", nil) + require.NoError(t, err) + request.Header.Set("Authorization", "Bearer "+jsonToken) + + resp, err := cli.Do(request) + require.NoError(t, err) + requireBodyContains(t, resp, "access denied") + require.Equal(t, http.StatusForbidden, resp.StatusCode) + }) + + t.Run("upload", func(t *testing.T) { + request, _, _ := makePutRequest(t, testHost+"/upload/"+cnrID.EncodeToString()) + request.Header.Set("Authorization", "Bearer "+jsonToken) + + resp, err := cli.Do(request) + require.NoError(t, err) + requireBodyContains(t, resp, "access denied") + require.Equal(t, http.StatusForbidden, resp.StatusCode) + }) + }) +} + +func requireBodyContains(t *testing.T, resp *http.Response, msg string) { + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + defer resp.Body.Close() + + require.Contains(t, strings.ToLower(string(data)), strings.ToLower(msg)) } func createDockerContainer(ctx context.Context, t *testing.T, image string) testcontainers.Container { @@ -485,6 +561,10 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool } func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, name string) (cid.ID, error) { + return createContainerBase(ctx, t, clientPool, ownerID, acl.PublicRWExtended, name) +} + +func createContainerBase(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, basicACL acl.Basic, name string) (cid.ID, error) { var policy netmap.PlacementPolicy err := policy.DecodeString("REP 1") require.NoError(t, err) @@ -492,24 +572,28 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o var cnr container.Container cnr.Init() cnr.SetPlacementPolicy(policy) - cnr.SetBasicACL(acl.PublicRWExtended) + cnr.SetBasicACL(basicACL) cnr.SetOwner(ownerID) container.SetCreationTime(&cnr, time.Now()) - var domain container.Domain - domain.SetName(name) + if name != "" { + var domain container.Domain + domain.SetName(name) - cnr.SetAttribute(containerv2.SysAttributeName, domain.Name()) - cnr.SetAttribute(containerv2.SysAttributeZone, domain.Zone()) + cnr.SetAttribute(containerv2.SysAttributeName, domain.Name()) + cnr.SetAttribute(containerv2.SysAttributeZone, domain.Zone()) + } - var waitPrm pool.WaitParams - waitPrm.SetTimeout(15 * time.Second) - waitPrm.SetPollInterval(3 * time.Second) - - var prm pool.PrmContainerPut - prm.SetContainer(cnr) - prm.SetWaitParams(waitPrm) + prm := pool.PrmContainerPut{ + ClientParams: client.PrmContainerPut{ + Container: &cnr, + }, + WaitParams: &pool.WaitParams{ + Timeout: 15 * time.Second, + PollInterval: 3 * time.Second, + }, + } CID, err := clientPool.PutContainer(ctx, prm) if err != nil { @@ -556,13 +640,18 @@ func registerUser(t *testing.T, ctx context.Context, aioContainer testcontainers require.NoError(t, err) } -func makeBearerTokens(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string) (jsonTokenBase64, binaryTokenBase64 string) { +func makeBearerTokens(t *testing.T, key *keys.PrivateKey, ownerID user.ID, version string, records ...*eacl.Record) (jsonTokenBase64, binaryTokenBase64 string) { tkn := new(bearer.Token) tkn.ForUser(ownerID) tkn.SetExp(10000) if version == "1.2.7" { - tkn.SetEACLTable(*eacl.NewTable()) + table := eacl.NewTable() + for i := range records { + table.AddRecord(records[i]) + } + + tkn.SetEACLTable(*table) } else { tkn.SetImpersonate(true) } From cb72d11515af5b8fbc602e8ca4e50578d68f7e3b Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Fri, 21 Mar 2025 13:38:43 +0300 Subject: [PATCH 538/548] [#224] Refactor logger tag configuration Signed-off-by: Pavel Pogodaev --- CHANGELOG.md | 1 + cmd/http-gw/app.go | 31 +++++++++++++++++++++++++++---- cmd/http-gw/logger.go | 19 +++++++++++-------- cmd/http-gw/settings.go | 13 +++++++++---- config/config.env | 5 +++-- config/config.yaml | 3 +-- docs/gate-configuration.md | 15 +++++++-------- 7 files changed, 59 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2025b6d..85798b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This document outlines major changes between releases. ### Added - Add handling quota limit reached error (#187) - Add slash clipping for FileName attribute (#174) +- Add new format of tag names config ## [0.32.3] - 2025-02-05 diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index de186fb..ca7797f 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -114,7 +114,8 @@ type ( } tagsConfig struct { - tagLogs sync.Map + tagLogs sync.Map + defaultLvl zap.AtomicLevel } logLevelConfig struct { @@ -134,19 +135,34 @@ func newLogLevel(v *viper.Viper) zap.AtomicLevel { } func newTagsConfig(v *viper.Viper, ll zapcore.Level) *tagsConfig { - var t tagsConfig + t := tagsConfig{defaultLvl: zap.NewAtomicLevelAt(ll)} if err := t.update(v, ll); err != nil { // panic here is analogue of the similar panic during common log level initialization. panic(err.Error()) } + return &t } func newLogLevelConfig(lvl zap.AtomicLevel, tagsConfig *tagsConfig) *logLevelConfig { - return &logLevelConfig{ + cfg := &logLevelConfig{ logLevel: lvl, tagsConfig: tagsConfig, } + + cfg.setMinLogLevel() + + return cfg +} + +func (l *logLevelConfig) setMinLogLevel() { + l.tagsConfig.tagLogs.Range(func(_, value any) bool { + v := value.(zapcore.Level) + if v < l.logLevel.Level() { + l.logLevel.SetLevel(v) + } + return true + }) } func (l *logLevelConfig) update(cfg *viper.Viper, log *zap.Logger) { @@ -159,17 +175,23 @@ func (l *logLevelConfig) update(cfg *viper.Viper, log *zap.Logger) { if err := l.tagsConfig.update(cfg, l.logLevel.Level()); err != nil { log.Warn(logs.TagsLogConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) } + + l.setMinLogLevel() } func (t *tagsConfig) LevelEnabled(tag string, tgtLevel zapcore.Level) bool { lvl, ok := t.tagLogs.Load(tag) if !ok { - return false + return t.defaultLvl.Enabled(tgtLevel) } return lvl.(zapcore.Level).Enabled(tgtLevel) } +func (t *tagsConfig) DefaultEnabled(lvl zapcore.Level) bool { + return t.defaultLvl.Enabled(lvl) +} + func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error { tags, err := fetchLogTagsConfig(cfg, ll) if err != nil { @@ -194,6 +216,7 @@ func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error { for k, v := range tags { t.tagLogs.Store(k, v) } + t.defaultLvl.SetLevel(ll) return nil } diff --git a/cmd/http-gw/logger.go b/cmd/http-gw/logger.go index 91105f7..195aa4e 100644 --- a/cmd/http-gw/logger.go +++ b/cmd/http-gw/logger.go @@ -40,7 +40,8 @@ type zapCoreTagFilterWrapper struct { } type TagFilterSettings interface { - LevelEnabled(tag string, lvl zapcore.Level) bool + LevelEnabled(tag string, tgtLevel zapcore.Level) bool + DefaultEnabled(lvl zapcore.Level) bool } func (c *zapCoreTagFilterWrapper) Enabled(level zapcore.Level) bool { @@ -63,24 +64,26 @@ func (c *zapCoreTagFilterWrapper) Check(entry zapcore.Entry, checked *zapcore.Ch } func (c *zapCoreTagFilterWrapper) Write(entry zapcore.Entry, fields []zapcore.Field) error { - if c.shouldSkip(entry, fields) || c.shouldSkip(entry, c.extra) { + if c.shouldSkip(entry, fields, c.extra) { return nil } return c.core.Write(entry, fields) } -func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field) bool { +func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field, extra []zap.Field) bool { for _, field := range fields { if field.Key == logs.TagFieldName && field.Type == zapcore.StringType { - if !c.settings.LevelEnabled(field.String, entry.Level) { - return true - } - break + return !c.settings.LevelEnabled(field.String, entry.Level) + } + } + for _, field := range extra { + if field.Key == logs.TagFieldName && field.Type == zapcore.StringType { + return !c.settings.LevelEnabled(field.String, entry.Level) } } - return false + return !c.settings.DefaultEnabled(entry.Level) } func (c *zapCoreTagFilterWrapper) Sync() error { diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 132c627..982b401 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -113,7 +113,7 @@ const ( cfgLoggerTags = "logger.tags" cfgLoggerTagsPrefixTmpl = cfgLoggerTags + ".%d." - cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "name" + cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "names" cfgLoggerTagsLevelTmpl = cfgLoggerTagsPrefixTmpl + "level" // Wallet. @@ -516,8 +516,8 @@ func fetchLogTagsConfig(v *viper.Viper, defaultLvl zapcore.Level) (map[string]za res := make(map[string]zapcore.Level) for i := 0; ; i++ { - name := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i)) - if name == "" { + tagNames := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i)) + if tagNames == "" { break } @@ -529,7 +529,12 @@ func fetchLogTagsConfig(v *viper.Viper, defaultLvl zapcore.Level) (map[string]za } } - res[name] = lvl + for _, tagName := range strings.Split(tagNames, ",") { + tagName = strings.TrimSpace(tagName) + if len(tagName) != 0 { + res[tagName] = lvl + } + } } if len(res) == 0 && !v.IsSet(cfgLoggerTags) { diff --git a/config/config.env b/config/config.env index 0ff2dec..72492d8 100644 --- a/config/config.env +++ b/config/config.env @@ -20,8 +20,9 @@ HTTP_GW_LOGGER_SAMPLING_ENABLED=false HTTP_GW_LOGGER_SAMPLING_INITIAL=100 HTTP_GW_LOGGER_SAMPLING_THEREAFTER=100 HTTP_GW_LOGGER_SAMPLING_INTERVAL=1s -HTTP_GW_LOGGER_TAGS_0_NAME=app -HTTP_GW_LOGGER_TAGS_1_NAME=datapath +HTTP_GW_LOGGER_TAGS_0_NAMES=app,datapath +HTTP_GW_LOGGER_TAGS_0_LEVEL=level +HTTP_GW_LOGGER_TAGS_1_NAME=external_storage_tree HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:443 HTTP_GW_SERVER_0_TLS_ENABLED=false diff --git a/config/config.yaml b/config/config.yaml index 05bba2e..ccd025e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -30,8 +30,7 @@ logger: thereafter: 100 interval: 1s tags: - - name: app - - name: datapath + - names: app,datapath level: debug server: diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 628d3c7..1dec574 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -176,10 +176,9 @@ logger: thereafter: 100 interval: 1s tags: - - name: "app" + - names: "app,datapath" level: info - - name: "datapath" - - name: "external_storage_tree" + - names: "external_storage_tree" ``` | Parameter | Type | SIGHUP reload | Default value | Description | @@ -199,14 +198,14 @@ parameter. Available tags: ```yaml tags: - - name: "app" + - names: "app,datapath" level: info ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-----------------------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------| -| `name` | `string` | yes | | Tag name. Possible values see below in `Tag values` section. | -| `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------| +| `names` | `[]string` | yes | | Tag names separated by `,`. Possible values see below in `Tag values` section. | +| `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. | ### Tag values From 273459e0904b97929143a0237d59222419bc59ee Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 7 Apr 2025 16:50:48 +0300 Subject: [PATCH 539/548] [#225] Support wildcard in allowed origins and headers Signed-off-by: Marina Biryukova --- internal/handler/cors.go | 29 +- internal/handler/cors_test.go | 490 ++++++++++++++++++++++++++++++++++ 2 files changed, 510 insertions(+), 9 deletions(-) diff --git a/internal/handler/cors.go b/internal/handler/cors.go index d77ae02..bbfce1e 100644 --- a/internal/handler/cors.go +++ b/internal/handler/cors.go @@ -5,6 +5,8 @@ import ( "encoding/xml" "errors" "fmt" + "regexp" + "slices" "sort" "strconv" "strings" @@ -78,7 +80,7 @@ func (h *Handler) Preflight(req *fasthttp.RequestCtx) { for _, rule := range corsConfig.CORSRules { for _, o := range rule.AllowedOrigins { - if o == string(origin) || o == wildcard { + if o == string(origin) || o == wildcard || (strings.Contains(o, "*") && match(o, string(origin))) { for _, m := range rule.AllowedMethods { if m == string(method) { if !checkSubslice(rule.AllowedHeaders, headers) { @@ -117,6 +119,11 @@ func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) { return } + method := req.Request.Header.Peek(fasthttp.HeaderAccessControlRequestMethod) + if len(method) == 0 { + method = req.Method() + } + ctx = qostagging.ContextWithIOTag(ctx, internalIOTag) cidParam, _ := req.UserValue("cid").(string) reqLog := h.reqLogger(ctx) @@ -141,9 +148,9 @@ func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) { for _, rule := range corsConfig.CORSRules { for _, o := range rule.AllowedOrigins { - if o == string(origin) { + if o == string(origin) || (strings.Contains(o, "*") && len(o) > 1 && match(o, string(origin))) { for _, m := range rule.AllowedMethods { - if m == string(req.Method()) { + if m == string(method) { req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) req.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") @@ -154,7 +161,7 @@ func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) { } if o == wildcard { for _, m := range rule.AllowedMethods { - if m == string(req.Method()) { + if m == string(method) { if withCredentials { req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") @@ -318,12 +325,9 @@ func setCORSHeadersFromRule(c *fasthttp.RequestCtx, cors *data.CORSRule) { } func checkSubslice(slice []string, subSlice []string) bool { - if sliceContains(slice, wildcard) { + if slices.Contains(slice, wildcard) { return true } - if len(subSlice) > len(slice) { - return false - } for _, r := range subSlice { if !sliceContains(slice, r) { return false @@ -334,9 +338,16 @@ func checkSubslice(slice []string, subSlice []string) bool { func sliceContains(slice []string, str string) bool { for _, s := range slice { - if s == str { + if s == str || (strings.Contains(s, "*") && match(s, str)) { return true } } return false } + +func match(tmpl, str string) bool { + regexpStr := "^" + regexp.QuoteMeta(tmpl) + "$" + regexpStr = regexpStr[:strings.Index(regexpStr, "*")-1] + "." + regexpStr[strings.Index(regexpStr, "*"):] + reg := regexp.MustCompile(regexpStr) + return reg.Match([]byte(str)) +} diff --git a/internal/handler/cors_test.go b/internal/handler/cors_test.go index 7cd7b0d..1ac07d7 100644 --- a/internal/handler/cors_test.go +++ b/internal/handler/cors_test.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/xml" "fmt" + "net/http" "testing" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" @@ -407,6 +408,12 @@ func TestCheckSubslice(t *testing.T) { actual: []string{"str1", "str5"}, expected: false, }, + { + name: "wildcard in allowed", + allowed: []string{"str*"}, + actual: []string{"str", "str5"}, + expected: true, + }, } { t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.expected, checkSubslice(tc.allowed, tc.actual)) @@ -414,6 +421,489 @@ func TestCheckSubslice(t *testing.T) { } } +func TestAllowedOriginWildcards(t *testing.T) { + hc := prepareHandlerContext(t) + bktName := "bucket-allowed-origin-wildcards" + cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private) + require.NoError(t, err) + hc.frostfs.SetContainer(cnrID, cnr) + + cfg := &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"*suffix.example"}, + AllowedMethods: []string{"GET"}, + }, + { + AllowedOrigins: []string{"https://*example"}, + AllowedMethods: []string{"GET"}, + }, + { + AllowedOrigins: []string{"prefix.example*"}, + AllowedMethods: []string{"GET"}, + }, + }, + } + setCORSObject(t, hc, cnrID, cfg, 1) + + for _, tc := range []struct { + name string + handler func(*fasthttp.RequestCtx) + requestHeaders map[string]string + expectedHeaders map[string]string + expectedStatus int + }{ + { + name: "set cors headers, empty request cors headers", + handler: hc.Handler().SetCORSHeaders, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, invalid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://origin.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, first rule, no symbols in place of wildcard", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "suffix.example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "suffix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, first rule, valid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://suffix.example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://suffix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, first rule, invalid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://suffix-example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, second rule, no symbols in place of wildcard", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, second rule, valid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://www.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, second rule, invalid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, third rule, no symbols in place of wildcard", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "prefix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, third rule, valid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example.com", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "set cors headers, third rule, invalid origin", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "www.prefix.example", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, third rule, invalid request method in header", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlRequestMethod: "PUT", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + }, + { + name: "set cors headers, third rule, valid request method in header", + handler: hc.Handler().SetCORSHeaders, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, empty request cors headers", + handler: hc.Handler().Preflight, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusBadRequest, + }, + { + name: "preflight, invalid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://origin.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "preflight, first rule, no symbols in place of wildcard", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "suffix.example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "suffix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "prelight, first rule, valid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://suffix.example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "http://suffix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, first rule, invalid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "http://suffix-example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "preflight, second rule, no symbols in place of wildcard", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, second rule, valid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://www.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, second rule, invalid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "preflight, third rule, no symbols in place of wildcard", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "prefix.example", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, third rule, valid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlAllowMethods: "GET", + }, + }, + { + name: "preflight, third rule, invalid origin", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "www.prefix.example", + fasthttp.HeaderAccessControlRequestMethod: "GET", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "preflight, third rule, invalid request method in header", + handler: hc.Handler().Preflight, + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "prefix.example.com", + fasthttp.HeaderAccessControlRequestMethod: "PUT", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + }, + expectedStatus: http.StatusForbidden, + }, + } { + t.Run(tc.name, func(t *testing.T) { + r := prepareCORSRequest(t, bktName, tc.requestHeaders) + tc.handler(r) + + expectedStatus := fasthttp.StatusOK + if tc.expectedStatus != 0 { + expectedStatus = tc.expectedStatus + } + require.Equal(t, expectedStatus, r.Response.StatusCode()) + for k, v := range tc.expectedHeaders { + require.Equal(t, v, string(r.Response.Header.Peek(k))) + } + }) + } +} + +func TestAllowedHeaderWildcards(t *testing.T) { + hc := prepareHandlerContext(t) + bktName := "bucket-allowed-header-wildcards" + cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private) + require.NoError(t, err) + hc.frostfs.SetContainer(cnrID, cnr) + + cfg := &data.CORSConfiguration{ + CORSRules: []data.CORSRule{ + { + AllowedOrigins: []string{"https://www.example.com"}, + AllowedMethods: []string{"HEAD"}, + AllowedHeaders: []string{"*-suffix"}, + }, + { + AllowedOrigins: []string{"https://www.example.com"}, + AllowedMethods: []string{"HEAD"}, + AllowedHeaders: []string{"start-*-end"}, + }, + { + AllowedOrigins: []string{"https://www.example.com"}, + AllowedMethods: []string{"HEAD"}, + AllowedHeaders: []string{"X-Amz-*"}, + }, + }, + } + setCORSObject(t, hc, cnrID, cfg, 1) + + for _, tc := range []struct { + name string + requestHeaders map[string]string + expectedHeaders map[string]string + expectedStatus int + }{ + { + name: "first rule, valid headers", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "header-suffix, -suffix", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlAllowMethods: "HEAD", + fasthttp.HeaderAccessControlAllowHeaders: "header-suffix, -suffix", + }, + }, + { + name: "first rule, invalid headers", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "header-suffix-*", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "second rule, valid headers", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "start--end, start-header-end", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlAllowMethods: "HEAD", + fasthttp.HeaderAccessControlAllowHeaders: "start--end, start-header-end", + }, + }, + { + name: "second rule, invalid header ending", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "start-header-end-*", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "second rule, invalid header beginning", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "*-start-header-end", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + }, + expectedStatus: http.StatusForbidden, + }, + { + name: "third rule, valid headers", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "X-Amz-Date, X-Amz-Content-Sha256", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlAllowMethods: "HEAD", + fasthttp.HeaderAccessControlAllowHeaders: "X-Amz-Date, X-Amz-Content-Sha256", + }, + }, + { + name: "third rule, invalid headers", + requestHeaders: map[string]string{ + fasthttp.HeaderOrigin: "https://www.example.com", + fasthttp.HeaderAccessControlRequestMethod: "HEAD", + fasthttp.HeaderAccessControlRequestHeaders: "Authorization", + }, + expectedHeaders: map[string]string{ + fasthttp.HeaderAccessControlAllowOrigin: "", + fasthttp.HeaderAccessControlAllowMethods: "", + fasthttp.HeaderAccessControlAllowHeaders: "", + }, + expectedStatus: http.StatusForbidden, + }, + } { + t.Run(tc.name, func(t *testing.T) { + r := prepareCORSRequest(t, bktName, tc.requestHeaders) + hc.Handler().Preflight(r) + + expectedStatus := http.StatusOK + if tc.expectedStatus != 0 { + expectedStatus = tc.expectedStatus + } + require.Equal(t, expectedStatus, r.Response.StatusCode()) + for k, v := range tc.expectedHeaders { + require.Equal(t, v, string(r.Response.Header.Peek(k))) + } + }) + } +} + func setCORSObject(t *testing.T, hc *handlerContext, cnrID cid.ID, corsConfig *data.CORSConfiguration, epoch uint64) { payload, err := xml.Marshal(corsConfig) require.NoError(t, err) From 304dbdd4c8deeec5850bc07ae5cd28c2acd0b26e Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Wed, 16 Apr 2025 16:39:26 +0300 Subject: [PATCH 540/548] [#228] Update Go to 1.23 Signed-off-by: Nikita Zinkevich --- .docker/Dockerfile | 2 +- .forgejo/workflows/builds.yml | 2 +- .forgejo/workflows/tests.yml | 6 +++--- .forgejo/workflows/vulncheck.yml | 2 +- .golangci.yml | 3 --- CHANGELOG.md | 2 ++ Makefile | 8 ++++---- go.mod | 2 +- 8 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index f45c864..8d6f806 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22-alpine AS basebuilder +FROM golang:1.24-alpine AS basebuilder RUN apk add --update make bash ca-certificates FROM basebuilder AS builder diff --git a/.forgejo/workflows/builds.yml b/.forgejo/workflows/builds.yml index 7c2bb04..ebb6bcc 100644 --- a/.forgejo/workflows/builds.yml +++ b/.forgejo/workflows/builds.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.22', '1.23' ] + go_versions: [ '1.23', '1.24' ] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index d4182ed..8fb4c10 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.23' + go-version: '1.24' cache: true - name: Install linters @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.22', '1.23' ] + go_versions: [ '1.23', '1.24' ] fail-fast: false steps: - uses: actions/checkout@v3 @@ -53,7 +53,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.23' + go-version: '1.24' - name: Run integration tests run: |- diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 5fb9dc5..a58d2df 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -16,7 +16,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.23' check-latest: true - name: Install govulncheck diff --git a/.golangci.yml b/.golangci.yml index d9f93eb..2c754ac 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -22,9 +22,6 @@ linters-settings: # 'default' case is present, even if all enum members aren't listed in the # switch default-signifies-exhaustive: true - govet: - # report about shadowed variables - check-shadowing: false custom: truecloudlab-linters: path: bin/external_linters.so diff --git a/CHANGELOG.md b/CHANGELOG.md index 85798b8..4465d2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ This document outlines major changes between releases. ## [Unreleased] +- Update Go to 1.23 (#228) + ### Added - Add handling quota limit reached error (#187) - Add slash clipping for FileName attribute (#174) diff --git a/Makefile b/Makefile index 5b9e5bf..2218765 100755 --- a/Makefile +++ b/Makefile @@ -2,9 +2,9 @@ REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") -GO_VERSION ?= 1.22 -LINT_VERSION ?= 1.60.3 -TRUECLOUDLAB_LINT_VERSION ?= 0.0.6 +GO_VERSION ?= 1.23 +LINT_VERSION ?= 1.64.8 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.10 BUILD ?= $(shell date -u --iso=seconds) HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs-http-gw @@ -150,7 +150,7 @@ dirty-image: @@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR) @rm -rf $(TMP_DIR)/linters @rmdir $(TMP_DIR) 2>/dev/null || true - @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) + @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install -trimpath github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) # Run linters lint: diff --git a/go.mod b/go.mod index 31cf242..c065b57 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw -go 1.22 +go 1.23 require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 From b9f1f455f8ce421ecb2f8ee5a24ffcf0d5026de2 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Wed, 16 Apr 2025 16:11:43 +0300 Subject: [PATCH 541/548] [#229] Add ngfuzz installation to makefile Signed-off-by: Marina Biryukova --- Makefile | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 2218765..11084f0 100755 --- a/Makefile +++ b/Makefile @@ -30,9 +30,10 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed "s/-/~/")-${OS_RELEASE} .PHONY: debpackage debclean -FUZZ_NGFUZZ_DIR ?= "" +FUZZING_DIR = $(shell pwd)/tests/fuzzing/files +NGFUZZ_REPO = https://gitflic.ru/project/yadro/ngfuzz.git FUZZ_TIMEOUT ?= 30 -FUZZ_FUNCTIONS ?= "all" +FUZZ_FUNCTIONS ?= "" FUZZ_AUX ?= "" # Make all binaries @@ -99,18 +100,22 @@ check-ngfuzz: exit 1; \ fi -.PHONY: install-fuzzing-deps -install-fuzzing-deps: check-clang check-ngfuzz +.PHONY: install-ngfuzz +install-ngfuzz: +ifeq (,$(wildcard $(FUZZING_DIR)/ngfuzz)) + @rm -rf $(FUZZING_DIR)/ngfuzz + @git clone $(NGFUZZ_REPO) $(FUZZING_DIR)/ngfuzz + @cd $(FUZZING_DIR)/ngfuzz && make +endif .PHONY: fuzz -fuzz: install-fuzzing-deps +fuzz: check-clang install-ngfuzz @START_PATH=$$(pwd); \ - ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \ - cd $(FUZZ_NGFUZZ_DIR) && \ - ./ngfuzz -clean && \ - ./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \ - ./ngfuzz -report - + ROOT_PATH=$$(realpath --relative-to=$(FUZZING_DIR)/ngfuzz $$START_PATH) ; \ + cd $(FUZZING_DIR)/ngfuzz && \ + ./bin/ngfuzz clean && \ + env CGO_ENABLED=1 ./bin/ngfuzz fuzz --funcs $(FUZZ_FUNCTIONS) --rootdir $$ROOT_PATH --timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \ + ./bin/ngfuzz coverage --rootdir $$ROOT_PATH # Reformat code fmt: From b7b08d9d828741ccad0169b32dae5c7230c27401 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Wed, 16 Apr 2025 17:53:49 +0300 Subject: [PATCH 542/548] [#230] Refactor logger tag configuration Signed-off-by: Pavel Pogodaev --- cmd/http-gw/app.go | 12 +++--------- cmd/http-gw/logger.go | 6 ++---- cmd/http-gw/settings.go | 1 - 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index ca7797f..ed16234 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -198,17 +198,11 @@ func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error { return err } - t.tagLogs.Range(func(key, value any) bool { + t.tagLogs.Range(func(key, _ any) bool { k := key.(string) - v := value.(zapcore.Level) - if lvl, ok := tags[k]; ok { - if lvl != v { - t.tagLogs.Store(key, lvl) - } - } else { + if _, ok := tags[k]; !ok { t.tagLogs.Delete(key) - delete(tags, k) } return true }) @@ -695,7 +689,7 @@ func (a *app) configReload(ctx context.Context) { return } - a.settings.logLevelConfig.update(a.cfg.settings, a.log) + a.settings.logLevelConfig.update(a.cfg.config(), a.log) if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil { a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) diff --git a/cmd/http-gw/logger.go b/cmd/http-gw/logger.go index 195aa4e..196cff3 100644 --- a/cmd/http-gw/logger.go +++ b/cmd/http-gw/logger.go @@ -40,7 +40,7 @@ type zapCoreTagFilterWrapper struct { } type TagFilterSettings interface { - LevelEnabled(tag string, tgtLevel zapcore.Level) bool + LevelEnabled(tag string, lvl zapcore.Level) bool DefaultEnabled(lvl zapcore.Level) bool } @@ -130,14 +130,13 @@ func newLogEncoder() zapcore.Encoder { // // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. func newStdoutLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) *Logger { - stdout := zapcore.AddSync(os.Stderr) + stdout := zapcore.AddSync(os.Stdout) consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, lvl) consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, loggerSettings, tagSetting) return &Logger{ logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), - lvl: lvl, } } @@ -155,7 +154,6 @@ func newJournaldLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings Logge return &Logger{ logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), - lvl: lvl, } } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 982b401..814a14e 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -208,7 +208,6 @@ var defaultTags = []string{logs.TagApp, logs.TagDatapath, logs.TagExternalStorag type Logger struct { logger *zap.Logger - lvl zap.AtomicLevel } type appCfg struct { From ee628617a36f3a7512a29414dabf7af173a06e5d Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Fri, 18 Apr 2025 14:34:16 +0300 Subject: [PATCH 543/548] [#227] Don't use bearer token with CORS container Signed-off-by: Marina Biryukova --- internal/handler/cors.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/handler/cors.go b/internal/handler/cors.go index bbfce1e..7e8db93 100644 --- a/internal/handler/cors.go +++ b/internal/handler/cors.go @@ -197,9 +197,6 @@ func (h *Handler) getCORSConfig(ctx context.Context, log *zap.Logger, cidStr str addr.SetContainer(h.corsCnrID) addr.SetObject(objID) corsObj, err := h.frostfs.GetObject(ctx, PrmObjectGet{ - PrmAuth: PrmAuth{ - BearerToken: bearerToken(ctx), - }, Address: addr, }) if err != nil { @@ -223,11 +220,7 @@ func (h *Handler) getLastCORSObject(ctx context.Context, cnrID cid.ID) (oid.ID, filters.AddRootFilter() filters.AddFilter(object.AttributeFilePath, fmt.Sprintf(corsFilePathTemplate, cnrID), object.MatchStringEqual) - prmAuth := PrmAuth{ - BearerToken: bearerToken(ctx), - } res, err := h.frostfs.SearchObjects(ctx, PrmObjectSearch{ - PrmAuth: prmAuth, Container: h.corsCnrID, Filters: filters, }) @@ -246,7 +239,6 @@ func (h *Handler) getLastCORSObject(ctx context.Context, cnrID cid.ID) (oid.ID, err = res.Iterate(func(id oid.ID) bool { addr.SetObject(id) obj, headErr = h.frostfs.HeadObject(ctx, PrmObjectHead{ - PrmAuth: prmAuth, Address: addr, }) if headErr != nil { From 9cb9d141463e6d3ad90826e6357cd6bc2d1b1655 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 22 Apr 2025 18:16:23 +0300 Subject: [PATCH 544/548] [#233] get/head: Middleware refactor Add: * search index.html * fallback by leading slash Signed-off-by: Denis Kirillov --- cmd/http-gw/app.go | 31 +++--- cmd/http-gw/settings.go | 5 +- config/config.env | 2 + config/config.yaml | 2 + docs/gate-configuration.md | 69 ++++++------- internal/handler/browse.go | 29 ++---- internal/handler/download.go | 165 +++++++++++++++++++++++++++++-- internal/handler/handler.go | 63 ++++++------ internal/handler/handler_test.go | 17 ++-- internal/handler/head.go | 45 ++++++++- internal/logs/logs.go | 4 +- 11 files changed, 311 insertions(+), 121 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index ed16234..f603d3b 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -100,17 +100,18 @@ type ( workerPoolSize int logLevelConfig *logLevelConfig - mu sync.RWMutex - defaultTimestamp bool - archiveCompression bool - clientCut bool - returnIndexPage bool - indexPageTemplate string - bufferMaxSizeForPut uint64 - namespaceHeader string - defaultNamespaces []string - cors *data.CORSRule - enableFilepathFallback bool + mu sync.RWMutex + defaultTimestamp bool + archiveCompression bool + clientCut bool + returnIndexPage bool + indexPageTemplate string + bufferMaxSizeForPut uint64 + namespaceHeader string + defaultNamespaces []string + cors *data.CORSRule + enableFilepathFallback bool + enableFilepathSlashFallback bool } tagsConfig struct { @@ -296,6 +297,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { indexPage, indexEnabled := fetchIndexPageTemplate(v, l) cors := fetchCORSConfig(v) enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback) + enableFilepathSlashFallback := v.GetBool(cfgFeaturesEnableFilepathSlashFallback) s.mu.Lock() defer s.mu.Unlock() @@ -311,6 +313,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { s.indexPageTemplate = indexPage s.cors = cors s.enableFilepathFallback = enableFilepathFallback + s.enableFilepathSlashFallback = enableFilepathSlashFallback } func (s *loggerSettings) DroppedLogsInc() { @@ -421,6 +424,12 @@ func (s *appSettings) EnableFilepathFallback() bool { return s.enableFilepathFallback } +func (s *appSettings) EnableFilepathSlashFallback() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.enableFilepathSlashFallback +} + func (a *app) initResolver() { var err error a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 814a14e..07722de 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -180,8 +180,9 @@ const ( cfgMultinetSubnets = "multinet.subnets" // Feature. - cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" - cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support" + cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" + cfgFeaturesEnableFilepathSlashFallback = "features.enable_filepath_slash_fallback" + cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support" // Containers. cfgContainersCORS = "containers.cors" diff --git a/config/config.env b/config/config.env index 72492d8..a86f3e8 100644 --- a/config/config.env +++ b/config/config.env @@ -174,6 +174,8 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl # Enable using fallback path to search for a object by attribute HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false +# See description in docs/gate-configuration.md +HTTP_GW_FEATURES_ENABLE_FILEPATH_SLASH_FALLBACK=false # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true diff --git a/config/config.yaml b/config/config.yaml index ccd025e..bb01d47 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -192,6 +192,8 @@ multinet: features: # Enable using fallback path to search for a object by attribute enable_filepath_fallback: false + # See description in docs/gate-configuration.md + enable_filepath_slash_fallback: false # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service tree_pool_netmap_support: true diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 1dec574..3a058ae 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -8,7 +8,6 @@ There are some custom types used for brevity: * `duration` -- string consisting of a number and a suffix. Suffix examples include `s` (seconds), `m` (minutes), `ms` ( milliseconds). - # Reload on SIGHUP Some config values can be reloaded on SIGHUP signal. @@ -163,7 +162,6 @@ server: | `tls.cert_file` | `string` | yes | | Path to the TLS certificate. | | `tls.key_file` | `string` | yes | | Path to the key. | - # `logger` section ```yaml @@ -177,7 +175,7 @@ logger: interval: 1s tags: - names: "app,datapath" - level: info + level: info - names: "external_storage_tree" ``` @@ -235,7 +233,6 @@ web: | `stream_request_body` | `bool` | `true` | Enables request body streaming, and calls the handler sooner when given body is larger than the current limit. | | `max_request_body_size` | `int` | `4194304` | Maximum request body size. The server rejects requests with bodies exceeding this limit. | - # `upload-header` section ```yaml @@ -271,7 +268,6 @@ archive: |---------------|--------|---------------|---------------|------------------------------------------------------------------| | `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. | - # `pprof` section Contains configuration for the `pprof` profiler. @@ -320,14 +316,13 @@ tracing: ``` | Parameter | Type | SIGHUP reload | Default value | Description | -| ------------ | -------------------------------------- | ------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------- | +|--------------|----------------------------------------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| | `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | | `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | | `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | | `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | | `attributes` | [[]Attributes](#attributes-subsection) | yes | | An array of configurable attributes in key-value format. | - #### `attributes` subsection ```yaml @@ -338,12 +333,13 @@ tracing: value: value ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-----------------------|----------|---------------|---------------|----------------------------------------------------------| -| `key` | `string` | yes | | Attribute key. | -| `value` | `string` | yes | | Attribute value. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|----------|---------------|---------------|------------------| +| `key` | `string` | yes | | Attribute key. | +| `value` | `string` | yes | | Attribute value. | # `runtime` section + Contains runtime parameters. ```yaml @@ -372,7 +368,6 @@ frostfs: | `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. | | `tree_pool_max_attempts` | `uint32` | no | `0` | Sets max attempt to make successful tree request. Value 0 means the number of attempts equals to number of nodes in pool. | - ### `cache` section ```yaml @@ -393,7 +388,6 @@ cache: | `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. | | `cors` | [Cache config](#cache-subsection) | `lifetime: 5m`
`size: 1000` | Cache which stores container CORS configurations. | - #### `cache` subsection ```yaml @@ -406,7 +400,6 @@ size: 1000 | `lifetime` | `duration` | depends on cache | Lifetime of entries in cache. | | `size` | `int` | depends on cache | LRU cache size. | - # `resolve_bucket` section Bucket name resolving parameters from and to container ID. @@ -417,10 +410,10 @@ resolve_bucket: default_namespaces: [ "", "root" ] ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|----------------------|------------|---------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------| -| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. | -| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|----------------------|------------|---------------|-----------------------|--------------------------------------------------| +| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. | +| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | # `index_page` section @@ -450,9 +443,9 @@ If values are not set, settings from CORS container will be used. ```yaml cors: allow_origin: "*" - allow_methods: ["GET", "HEAD"] - allow_headers: ["Authorization"] - expose_headers: ["*"] + allow_methods: [ "GET", "HEAD" ] + allow_headers: [ "Authorization" ] + expose_headers: [ "*" ] allow_credentials: false max_age: 600 ``` @@ -472,15 +465,15 @@ Configuration of multinet support. ```yaml multinet: - enabled: false - balancer: roundrobin - restrict: false - fallback_delay: 300ms - subnets: - - mask: 1.2.3.4/24 - source_ips: - - 1.2.3.4 - - 1.2.3.5 + enabled: false + balancer: roundrobin + restrict: false + fallback_delay: 300ms + subnets: + - mask: 1.2.3.4/24 + source_ips: + - 1.2.3.4 + - 1.2.3.5 ``` | Parameter | Type | SIGHUP reload | Default value | Description | @@ -512,13 +505,15 @@ Contains parameters for enabling features. ```yaml features: enable_filepath_fallback: true + enable_filepath_slash_fallback: false tree_pool_netmap_support: true ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. | -| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-------------------------------------------|--------|---------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FileName` attribute if object with `FilePath` attribute wasn't found. | +| `features.enable_filepath_slash_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FilePath`/`FileName` with/without (depends on provided value in `FilePath`/`FileName`) if object with provided `FilePath`/`FileName` wasn't found. This fallback goes `before enable_filepath_fallback`. | +| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | # `containers` section @@ -529,6 +524,6 @@ containers: cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-------------|----------|---------------|---------------|-----------------------------------------| -| `cors` | `string` | no | | Container name for CORS configurations. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|----------|---------------|---------------|-----------------------------------------| +| `cors` | `string` | no | | Container name for CORS configurations. | diff --git a/internal/handler/browse.go b/internal/handler/browse.go index ebe9004..e1fc59d 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -161,6 +160,7 @@ func urlencode(path string) string { type GetObjectsResponse struct { objects []ResponseObject hasErrors bool + isNative bool } func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { @@ -226,7 +226,8 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck log := h.reqLogger(ctx) dirs := make(map[string]struct{}) result := &GetObjectsResponse{ - objects: make([]ResponseObject, 0, 100), + objects: make([]ResponseObject, 0, 100), + isNative: true, } for objExt := range resp { if objExt.Error != nil { @@ -322,28 +323,16 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID, } type browseParams struct { - bucketInfo *data.BucketInfo - prefix string - isNative bool - listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) (*GetObjectsResponse, error) + bucketInfo *data.BucketInfo + prefix string + objects *GetObjectsResponse } func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p browseParams) { const S3Protocol = "s3" const FrostfsProtocol = "frostfs" - ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( - zap.String("bucket", p.bucketInfo.Name), - zap.String("container", p.bucketInfo.CID.EncodeToString()), - zap.String("prefix", p.prefix), - )) - resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix) - if err != nil { - h.logAndSendError(ctx, req, logs.FailedToListObjects, err) - return - } - - objects := resp.objects + objects := p.objects.objects sort.Slice(objects, func(i, j int) bool { if objects[i].IsDir == objects[j].IsDir { return objects[i].FileName < objects[j].FileName @@ -363,7 +352,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p } bucketName := p.bucketInfo.Name protocol := S3Protocol - if p.isNative { + if p.objects.isNative { bucketName = p.bucketInfo.CID.EncodeToString() protocol = FrostfsProtocol } @@ -372,7 +361,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p Prefix: p.prefix, Objects: objects, Protocol: protocol, - HasErrors: resp.hasErrors, + HasErrors: p.objects.hasErrors, }); err != nil { h.logAndSendError(ctx, req, logs.FailedToExecuteTemplate, err) return diff --git a/internal/handler/download.go b/internal/handler/download.go index 114bf34..301d10f 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "net/url" + "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" @@ -31,13 +32,18 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { cidParam := req.UserValue("cid").(string) oidParam := req.UserValue("oid").(string) - downloadParam := req.QueryArgs().GetBool("download") ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( zap.String("cid", cidParam), zap.String("oid", oidParam), )) + path, err := url.QueryUnescape(oidParam) + if err != nil { + h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err) + return + } + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) @@ -50,18 +56,159 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { return } - var objID oid.ID - if checkS3Err == nil && shouldDownload(oidParam, downloadParam) { - h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile) - } else if err = objID.DecodeString(oidParam); err == nil { - h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile) + prm := MiddlewareParam{ + Context: ctx, + Request: req, + BktInfo: bktInfo, + Path: path, + } + + indexPageEnabled := h.config.IndexPageEnabled() + + if checkS3Err == nil { + run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound), + Middleware{Func: h.byS3PathMiddleware(h.receiveFile, noopFormer), Enabled: true}, + Middleware{Func: h.byS3PathMiddleware(h.receiveFile, indexFormer), Enabled: indexPageEnabled}, + Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsS3), Enabled: indexPageEnabled}, + ) } else { - h.browseIndex(ctx, req, cidParam, oidParam, checkS3Err != nil) + slashFallbackEnabled := h.config.EnableFilepathSlashFallback() + fileNameFallbackEnabled := h.config.EnableFilepathFallback() + + run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound), + Middleware{Func: h.byAddressMiddleware(h.receiveFile), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, noopFormer), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled}, + Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsNative), Enabled: indexPageEnabled}, + ) } } -func shouldDownload(oidParam string, downloadParam bool) bool { - return !isDir(oidParam) || downloadParam +type MiddlewareFunc func(param MiddlewareParam) bool + +type MiddlewareParam struct { + Context context.Context + Request *fasthttp.RequestCtx + BktInfo *data.BucketInfo + Path string +} + +type Middleware struct { + Func MiddlewareFunc + Enabled bool +} + +func run(prm MiddlewareParam, defaultMiddleware MiddlewareFunc, middlewares ...Middleware) { + for _, m := range middlewares { + if m.Enabled && !m.Func(prm) { + return + } + } + + defaultMiddleware(prm) +} + +func indexFormer(path string) string { + indexPath := path + if indexPath != "" && !strings.HasSuffix(indexPath, "/") { + indexPath += "/" + } + + return indexPath + "index.html" +} + +func reverseLeadingSlash(path string) string { + if path == "" || path == "/" { + return path + } + + if path[0] == '/' { + return path[1:] + } + + return "/" + path +} + +func noopFormer(path string) string { + return path +} + +func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), pathFormer func(string) string) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byS3Path") + defer span.End() + + path := pathFormer(prm.Path) + + foundOID, err := h.tree.GetLatestVersion(ctx, &prm.BktInfo.CID, path) + if err == nil { + if foundOID.IsDeleteMarker { + h.logAndSendError(ctx, prm.Request, logs.IndexWasDeleted, ErrObjectNotFound) + return false + } + + addr := newAddress(prm.BktInfo.CID, foundOID.OID) + handler(ctx, prm.Request, addr) + return false + } + + if !errors.Is(err, layer.ErrNodeNotFound) { + h.logAndSendError(ctx, prm.Request, logs.FailedToGetLatestVersionOfIndexObject, err, zap.String("path", path)) + return false + } + + return true + } +} + +func (h *Handler) byAttributeSearchMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), attr string, pathFormer func(string) string) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAttributeSearch") + defer span.End() + + path := pathFormer(prm.Path) + + res, err := h.search(ctx, prm.BktInfo.CID, attr, path, object.MatchStringEqual) + if err != nil { + h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err) + return false + } + defer res.Close() + + buf := make([]oid.ID, 1) + n, err := res.Read(buf) + if err == nil && n > 0 { + addr := newAddress(prm.BktInfo.CID, buf[0]) + handler(ctx, prm.Request, addr) + return false + } + + if !errors.Is(err, io.EOF) { + h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err) + return false + } + + return true + } +} + +func (h *Handler) byAddressMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAddress") + defer span.End() + + var objID oid.ID + if objID.DecodeString(prm.Path) == nil { + handler(ctx, prm.Request, newAddress(prm.BktInfo.CID, objID)) + return false + } + + return true + } } // DownloadByAttribute handles attribute-based download requests. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index a982bc2..b0daf44 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -35,6 +35,7 @@ type Config interface { BufferMaxSizeForPut() uint64 NamespaceHeader() string EnableFilepathFallback() bool + EnableFilepathSlashFallback() bool FormContainerZone(string) string CORS() *data.CORSRule } @@ -216,11 +217,11 @@ func (h *Handler) byNativeAddress(ctx context.Context, req *fasthttp.RequestCtx, // byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that // resolves object address from S3-like path /. -func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { +func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, bktInfo *data.BucketInfo, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path") defer span.End() - foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) + foundOID, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, path) if err != nil { h.logAndSendError(ctx, req, logs.FailedToGetLatestVersionOfObject, err, zap.String("path", path)) return @@ -230,7 +231,7 @@ func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID return } - addr := newAddress(cnrID, foundOID.OID) + addr := newAddress(bktInfo.CID, foundOID.OID) handler(ctx, req, addr) } @@ -418,37 +419,31 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } -func (h *Handler) browseIndex(ctx context.Context, req *fasthttp.RequestCtx, cidParam, oidParam string, isNativeList bool) { - ctx, span := tracing.StartSpanFromContext(ctx, "handler.browseIndex") - defer span.End() +type ListFunc func(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) - if !h.config.IndexPageEnabled() { - req.SetStatusCode(fasthttp.StatusNotFound) - return +func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex") + defer span.End() + + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( + zap.String("bucket", prm.BktInfo.Name), + zap.String("container", prm.BktInfo.CID.EncodeToString()), + zap.String("prefix", prm.Path), + )) + + objects, err := fn(ctx, prm.BktInfo, prm.Path) + if err != nil { + h.logAndSendError(ctx, prm.Request, logs.FailedToListObjects, err) + return false + } + + h.browseObjects(ctx, prm.Request, browseParams{ + bucketInfo: prm.BktInfo, + prefix: prm.Path, + objects: objects, + }) + + return false } - - unescapedKey, err := url.QueryUnescape(oidParam) - if err != nil { - h.logAndSendError(ctx, req, logs.FailedToUnescapeOIDParam, err) - return - } - - bktInfo, err := h.getBucketInfo(ctx, cidParam) - if err != nil { - h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) - return - } - - listFunc := h.getDirObjectsS3 - if isNativeList { - // tree probe failed, trying to use native - listFunc = h.getDirObjectsNative - } - - h.browseObjects(ctx, req, browseParams{ - bucketInfo: bktInfo, - prefix: unescapedKey, - listObjects: listFunc, - isNative: isNativeList, - }) } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 93cb1d9..fc75d69 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -62,8 +62,9 @@ func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*d } type configMock struct { - additionalSearch bool - cors *data.CORSRule + additionalFilenameSearch bool + additionalSlashSearch bool + cors *data.CORSRule } func (c *configMock) DefaultTimestamp() bool { @@ -99,7 +100,11 @@ func (c *configMock) NamespaceHeader() string { } func (c *configMock) EnableFilepathFallback() bool { - return c.additionalSearch + return c.additionalFilenameSearch +} + +func (c *configMock) EnableFilepathSlashFallback() bool { + return c.additionalSlashSearch } func (c *configMock) FormContainerZone(string) string { @@ -327,7 +332,7 @@ func TestBasic(t *testing.T) { func TestFindObjectByAttribute(t *testing.T) { hc := prepareHandlerContext(t) - hc.cfg.additionalSearch = true + hc.cfg.additionalFilenameSearch = true bktName := "bucket" cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended) @@ -407,7 +412,7 @@ func TestFindObjectByAttribute(t *testing.T) { t.Run(tc.name, func(t *testing.T) { obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] obj.SetAttributes(tc.firstAttr, tc.secondAttr) - hc.cfg.additionalSearch = tc.additionalSearch + hc.cfg.additionalFilenameSearch = tc.additionalSearch objID, err := hc.Handler().findObjectByAttribute(ctx, cnrID, tc.reqAttrKey, tc.reqAttrValue) if tc.err != "" { @@ -476,7 +481,7 @@ func TestNeedSearchByFileName(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - hc.cfg.additionalSearch = tc.additionalSearch + hc.cfg.additionalFilenameSearch = tc.additionalSearch res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal) require.Equal(t, tc.expected, res) diff --git a/internal/handler/head.go b/internal/handler/head.go index 11d45fc..e130124 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -5,6 +5,7 @@ import ( "errors" "io" "net/http" + "net/url" "strconv" "time" @@ -128,6 +129,12 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { zap.String("oid", oidParam), )) + path, err := url.QueryUnescape(oidParam) + if err != nil { + h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err) + return + } + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) @@ -140,9 +147,38 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { return } + prm := MiddlewareParam{ + Context: ctx, + Request: req, + BktInfo: bktInfo, + Path: path, + } + + indexPageEnabled := h.config.IndexPageEnabled() + + if checkS3Err == nil { + run(prm, h.errorMiddleware(logs.ObjectNotFound, layer.ErrNodeNotFound), + Middleware{Func: h.byS3PathMiddleware(h.headObject, noopFormer), Enabled: true}, + Middleware{Func: h.byS3PathMiddleware(h.headObject, indexFormer), Enabled: indexPageEnabled}, + ) + } else { + slashFallbackEnabled := h.config.EnableFilepathSlashFallback() + fileNameFallbackEnabled := h.config.EnableFilepathFallback() + + run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound), + Middleware{Func: h.byAddressMiddleware(h.headObject), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, noopFormer), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled}, + ) + } + var objID oid.ID if checkS3Err == nil { - h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject) + h.byS3Path(ctx, req, bktInfo, oidParam, h.headObject) } else if err = objID.DecodeString(oidParam); err == nil { h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) } else { @@ -157,3 +193,10 @@ func (h *Handler) HeadByAttribute(req *fasthttp.RequestCtx) { h.byAttribute(ctx, req, h.headObject) } + +func (h *Handler) errorMiddleware(msg string, err error) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + h.logAndSendError(prm.Context, prm.Request, msg, err) + return false + } +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 3e9b931..213e7c7 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -108,7 +108,9 @@ const ( FailedToGetBucketInfo = "could not get bucket info" FailedToSubmitTaskToPool = "failed to submit task to pool" ObjectWasDeleted = "object was deleted" + IndexWasDeleted = "index was deleted" FailedToGetLatestVersionOfObject = "failed to get latest version of object" + FailedToGetLatestVersionOfIndexObject = "failed to get latest version of index object" FailedToCheckIfSettingsNodeExist = "failed to check if settings node exists" FailedToListObjects = "failed to list objects" FailedToParseTemplate = "failed to parse template" @@ -118,7 +120,7 @@ const ( FailedToGetObject = "failed to get object" FailedToGetObjectPayload = "failed to get object payload" FailedToFindObjectByAttribute = "failed to get find object by attribute" - FailedToUnescapeOIDParam = "failed to unescape oid param" + FailedToUnescapePath = "failed to unescape path" InvalidOIDParam = "invalid oid param" CouldNotGetCORSConfiguration = "could not get cors configuration" EmptyOriginRequestHeader = "empty Origin request header" From 0b9b23e67c2daf35bda7254610ce0d95d2233301 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 23 Apr 2025 09:18:21 +0300 Subject: [PATCH 545/548] [#233] Make search by attribute as it is Signed-off-by: Denis Kirillov --- internal/handler/handler.go | 42 ---------------- internal/handler/handler_test.go | 84 -------------------------------- internal/logs/logs.go | 7 ++- 3 files changed, 3 insertions(+), 130 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index b0daf44..59a19ed 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -253,8 +253,6 @@ func (h *Handler) byAttribute(ctx context.Context, req *fasthttp.RequestCtx, han return } - val = prepareAtribute(key, val) - ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val))) @@ -292,10 +290,6 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, cnrID cid.ID, attrK n, err := res.Read(buf) if n == 0 { switch { - case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): - h.reqLogger(ctx).Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) - attrVal = prepareAtribute(attrFileName, attrVal) - return h.findObjectByAttribute(ctx, cnrID, attrFileName, attrVal) case errors.Is(err, io.EOF): h.reqLogger(ctx).Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("object not found: %w", err) @@ -308,42 +302,6 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, cnrID cid.ID, attrK return buf[0], nil } -func (h *Handler) needSearchByFileName(key, val string) bool { - if key != attrFilePath || !h.config.EnableFilepathFallback() { - return false - } - - return strings.HasPrefix(val, "/") && strings.Count(val, "/") == 1 || !strings.Contains(val, "/") -} - -func prepareAtribute(attrKey, attrVal string) string { - if attrKey == attrFileName { - return prepareFileName(attrVal) - } - - if attrKey == attrFilePath { - return prepareFilePath(attrVal) - } - - return attrVal -} - -func prepareFileName(fileName string) string { - if strings.HasPrefix(fileName, "/") { - return fileName[1:] - } - - return fileName -} - -func prepareFilePath(filePath string) string { - if !strings.HasPrefix(filePath, "/") { - return "/" + filePath - } - - return filePath -} - // resolveContainer decode container id, if it's not a valid container id // then trey to resolve name using provided resolver. func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index fc75d69..81d9784 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -427,90 +427,6 @@ func TestFindObjectByAttribute(t *testing.T) { } } -func TestNeedSearchByFileName(t *testing.T) { - hc := prepareHandlerContext(t) - - for _, tc := range []struct { - name string - attrKey string - attrVal string - additionalSearch bool - expected bool - }{ - { - name: "need search - not contains slash", - attrKey: attrFilePath, - attrVal: "cat.png", - additionalSearch: true, - expected: true, - }, - { - name: "need search - single lead slash", - attrKey: attrFilePath, - attrVal: "/cat.png", - additionalSearch: true, - expected: true, - }, - { - name: "don't need search - single slash but not lead", - attrKey: attrFilePath, - attrVal: "cats/cat.png", - additionalSearch: true, - expected: false, - }, - { - name: "don't need search - more one slash", - attrKey: attrFilePath, - attrVal: "/cats/cat.png", - additionalSearch: true, - expected: false, - }, - { - name: "don't need search - incorrect attribute key", - attrKey: attrFileName, - attrVal: "cat.png", - additionalSearch: true, - expected: false, - }, - { - name: "don't need search - additional search disabled", - attrKey: attrFilePath, - attrVal: "cat.png", - additionalSearch: false, - expected: false, - }, - } { - t.Run(tc.name, func(t *testing.T) { - hc.cfg.additionalFilenameSearch = tc.additionalSearch - - res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal) - require.Equal(t, tc.expected, res) - }) - } -} - -func TestPrepareFileName(t *testing.T) { - fileName := "/cat.jpg" - expected := "cat.jpg" - actual := prepareFileName(fileName) - require.Equal(t, expected, actual) - - fileName = "cat.jpg" - actual = prepareFileName(fileName) - require.Equal(t, expected, actual) -} - -func TestPrepareFilePath(t *testing.T) { - filePath := "cat.jpg" - expected := "/cat.jpg" - actual := prepareFilePath(filePath) - require.Equal(t, expected, actual) - - filePath = "/cat.jpg" - actual = prepareFilePath(filePath) - require.Equal(t, expected, actual) -} - func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 213e7c7..e7d118f 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -131,10 +131,9 @@ const ( // Log messages with the "external_storage" tag. const ( - ObjectNotFound = "object not found" - ReadObjectListFailed = "read object list failed" - ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" - ObjectUploaded = "object uploaded" + ObjectNotFound = "object not found" + ReadObjectListFailed = "read object list failed" + ObjectUploaded = "object uploaded" ) // Log messages with the "external_storage_tree" tag. From e579549b41fd1f35824968a331143a48f1204550 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 23 Apr 2025 10:52:19 +0300 Subject: [PATCH 546/548] [#233] Add fallback tests Signed-off-by: Denis Kirillov --- internal/handler/handler_test.go | 281 +++++++++++++++++++++---------- internal/handler/head.go | 9 - 2 files changed, 193 insertions(+), 97 deletions(-) diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 81d9784..dbb037d 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -26,6 +26,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/panjf2000/ants/v2" @@ -64,6 +65,7 @@ func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*d type configMock struct { additionalFilenameSearch bool additionalSlashSearch bool + indexEnabled bool cors *data.CORSRule } @@ -76,7 +78,7 @@ func (c *configMock) ArchiveCompression() bool { } func (c *configMock) IndexPageEnabled() bool { - return false + return c.indexEnabled } func (c *configMock) IndexPageTemplate() string { @@ -259,6 +261,7 @@ func TestBasic(t *testing.T) { err = json.Unmarshal(r.Response.Body(), &putRes) require.NoError(t, err) + hc.cfg.additionalFilenameSearch = true obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] fileName := prepareObjectAttributes(object.AttributeFileName, objFileName) filePath := prepareObjectAttributes(object.AttributeFilePath, objFilePath) @@ -269,6 +272,14 @@ func TestBasic(t *testing.T) { r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID) hc.Handler().DownloadByAddressOrBucketName(r) require.Equal(t, content, string(r.Response.Body())) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFilePath) + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, content, string(r.Response.Body())) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFileName) + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, content, string(r.Response.Body())) }) t.Run("head", func(t *testing.T) { @@ -276,6 +287,16 @@ func TestBasic(t *testing.T) { hc.Handler().HeadByAddressOrBucketName(r) require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFilePath) + hc.Handler().HeadByAddressOrBucketName(r) + require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) + require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFileName) + hc.Handler().HeadByAddressOrBucketName(r) + require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) + require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) }) t.Run("get by attribute", func(t *testing.T) { @@ -285,11 +306,11 @@ func TestBasic(t *testing.T) { r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath) hc.Handler().DownloadByAttribute(r) - require.Equal(t, content, string(r.Response.Body())) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName) hc.Handler().DownloadByAttribute(r) - require.Equal(t, content, string(r.Response.Body())) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) }) t.Run("head by attribute", func(t *testing.T) { @@ -300,13 +321,11 @@ func TestBasic(t *testing.T) { r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath) hc.Handler().HeadByAttribute(r) - require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) - require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName) hc.Handler().HeadByAttribute(r) - require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) - require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) }) t.Run("zip", func(t *testing.T) { @@ -330,101 +349,187 @@ func TestBasic(t *testing.T) { }) } -func TestFindObjectByAttribute(t *testing.T) { +func prepareHandlerAndBucket(t *testing.T) (*handlerContext, cid.ID) { hc := prepareHandlerContext(t) - hc.cfg.additionalFilenameSearch = true bktName := "bucket" cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended) require.NoError(t, err) hc.frostfs.SetContainer(cnrID, cnr) - ctx := context.Background() - ctx = middleware.SetNamespace(ctx, "") + return hc, cnrID +} - content := "hello" - r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content) - require.NoError(t, err) +func TestGetObjectWithFallback(t *testing.T) { + ctx := middleware.SetNamespace(context.Background(), "") - hc.Handler().Upload(r) - require.Equal(t, r.Response.StatusCode(), http.StatusOK) + t.Run("by oid", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) - var putRes putResponse - err = json.Unmarshal(r.Response.Body(), &putRes) - require.NoError(t, err) + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 - testAttrVal1 := "/folder/cat.jpg" - testAttrVal2 := "cat.jpg" - testAttrVal3 := "test-attr-val3" + r := prepareGetRequest(ctx, cnrID.EncodeToString(), obj1ID.String()) + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) - for _, tc := range []struct { - name string - firstAttr object.Attribute - secondAttr object.Attribute - reqAttrKey string - reqAttrValue string - err string - additionalSearch bool - }{ - { - name: "success search by FileName", - firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), - secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), - reqAttrKey: attrFileName, - reqAttrValue: testAttrVal2, - additionalSearch: false, - }, - { - name: "failed search by FileName", - firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), - secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), - reqAttrKey: attrFileName, - reqAttrValue: testAttrVal3, - err: "not found", - additionalSearch: false, - }, - { - name: "success search by FilePath (with additional search)", - firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), - secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), - reqAttrKey: attrFilePath, - reqAttrValue: testAttrVal2, - additionalSearch: true, - }, - { - name: "failed by FilePath (with additional search)", - firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), - secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), - reqAttrKey: attrFilePath, - reqAttrValue: testAttrVal3, - err: "not found", - additionalSearch: true, - }, - { - name: "success search by FilePath with leading slash (with additional search)", - firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1), - secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2), - reqAttrKey: attrFilePath, - reqAttrValue: "/cat.jpg", - additionalSearch: true, - }, - } { - t.Run(tc.name, func(t *testing.T) { - obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] - obj.SetAttributes(tc.firstAttr, tc.secondAttr) - hc.cfg.additionalFilenameSearch = tc.additionalSearch + t.Run("by filepath as it is", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) - objID, err := hc.Handler().findObjectByAttribute(ctx, cnrID, tc.reqAttrKey, tc.reqAttrValue) - if tc.err != "" { - require.Error(t, err) - require.Contains(t, err.Error(), tc.err) - return - } + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 - require.NoError(t, err) - require.Equal(t, putRes.ObjectID, objID.EncodeToString()) - }) - } + obj2ID := oidtest.ID() + obj2 := object.New() + obj2.SetID(obj2ID) + obj2.SetPayload([]byte("obj2")) + obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/filepath/obj2")) + hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj2") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj2.Payload()), string(r.Response.Body())) + }) + + t.Run("by filepath slash fallback", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.additionalSlashSearch = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) + + t.Run("by filename fallback", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.additionalFilenameSearch = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) + + t.Run("by filename and slash fallback", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.additionalFilenameSearch = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.additionalSlashSearch = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) + + t.Run("index fallback", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/index.html")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.indexEnabled = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) + + t.Run("index filename fallback", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/index.html")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.indexEnabled = true + hc.cfg.additionalFilenameSearch = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, string(obj1.Payload()), string(r.Response.Body())) + }) } func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { diff --git a/internal/handler/head.go b/internal/handler/head.go index e130124..e6d9a30 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -175,15 +175,6 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled}, ) } - - var objID oid.ID - if checkS3Err == nil { - h.byS3Path(ctx, req, bktInfo, oidParam, h.headObject) - } else if err = objID.DecodeString(oidParam); err == nil { - h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) - } else { - h.logAndSendError(ctx, req, logs.InvalidOIDParam, err) - } } // HeadByAttribute handles attribute-based head requests. From dbb1bcad00cf6a444e8a951d61755fd3766c481e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 23 Apr 2025 13:02:17 +0300 Subject: [PATCH 547/548] [#233] Fix browsing Simplify tree listing (we need only nodes in exactly the same parent level) Signed-off-by: Denis Kirillov --- docs/gate-configuration.md | 10 +- internal/handler/browse.go | 29 +++- internal/handler/download.go | 12 +- internal/handler/handler.go | 36 +---- internal/handler/handler_test.go | 131 ++++++++++++---- internal/handler/head.go | 6 +- .../handler/tree_service_client_mock_test.go | 141 ++++++++++++++++++ internal/handler/utils.go | 6 +- internal/layer/tree_service.go | 24 --- internal/templates/index.gotmpl | 12 +- tree/tree.go | 46 ++---- 11 files changed, 302 insertions(+), 151 deletions(-) create mode 100644 internal/handler/tree_service_client_mock_test.go delete mode 100644 internal/layer/tree_service.go diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 3a058ae..08e2679 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -509,11 +509,11 @@ features: tree_pool_netmap_support: true ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-------------------------------------------|--------|---------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FileName` attribute if object with `FilePath` attribute wasn't found. | -| `features.enable_filepath_slash_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FilePath`/`FileName` with/without (depends on provided value in `FilePath`/`FileName`) if object with provided `FilePath`/`FileName` wasn't found. This fallback goes `before enable_filepath_fallback`. | -| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-------------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FileName` attribute if object with `FilePath` attribute wasn't found. | +| `features.enable_filepath_slash_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FilePath`/`FileName` with/without (depends on provided value in `FilePath`/`FileName`) leading slash if object with provided `FilePath`/`FileName` wasn't found. This fallback goes before `enable_filepath_fallback`. | +| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | # `containers` section diff --git a/internal/handler/browse.go b/internal/handler/browse.go index e1fc59d..d9e6625 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -130,11 +130,15 @@ func parentDir(prefix string) string { return prefix[index:] } -func trimPrefix(encPrefix string) string { +func getParent(encPrefix string) string { prefix, err := url.PathUnescape(encPrefix) if err != nil { return "" } + if prefix != "" && prefix[len(prefix)-1] == '/' { + prefix = prefix[:len(prefix)-1] + } + slashIndex := strings.LastIndex(prefix, "/") if slashIndex == -1 { return "" @@ -164,7 +168,11 @@ type GetObjectsResponse struct { } func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { - nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) + if prefix != "" && prefix[len(prefix)-1] == '/' { + prefix = prefix[:len(prefix)-1] + } + + nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) if err != nil { return nil, err } @@ -185,7 +193,7 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn if obj.IsDeleteMarker { continue } - obj.FilePath = prefix + obj.FileName + obj.FilePath = prefix + "/" + obj.FileName obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath) result.objects = append(result.objects, obj) } @@ -194,9 +202,9 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn } func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { - var basePath string - if ind := strings.LastIndex(prefix, "/"); ind != -1 { - basePath = prefix[:ind+1] + basePath := prefix + if basePath != "" && basePath[len(basePath)-1] != '/' { + basePath += "/" } filters := object.NewSearchFilters() @@ -342,7 +350,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p tmpl, err := template.New("index").Funcs(template.FuncMap{ "formatSize": formatSize, - "trimPrefix": trimPrefix, + "getParent": getParent, "urlencode": urlencode, "parentDir": parentDir, }).Parse(h.config.IndexPageTemplate()) @@ -356,9 +364,14 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p bucketName = p.bucketInfo.CID.EncodeToString() protocol = FrostfsProtocol } + prefix := p.prefix + if prefix != "" && prefix[len(prefix)-1] != '/' { + prefix += "/" + } + if err = tmpl.Execute(req, &BrowsePageData{ Container: bucketName, - Prefix: p.prefix, + Prefix: prefix, Objects: objects, Protocol: protocol, HasErrors: p.objects.hasErrors, diff --git a/internal/handler/download.go b/internal/handler/download.go index 301d10f..15fb886 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -14,8 +14,8 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -51,7 +51,7 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { } checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) - if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) { h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) return } @@ -88,6 +88,8 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { } } +type ObjectHandlerFunc func(context.Context, *fasthttp.RequestCtx, oid.Address) + type MiddlewareFunc func(param MiddlewareParam) bool type MiddlewareParam struct { @@ -156,7 +158,7 @@ func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.Req return false } - if !errors.Is(err, layer.ErrNodeNotFound) { + if !errors.Is(err, tree.ErrNodeNotFound) { h.logAndSendError(ctx, prm.Request, logs.FailedToGetLatestVersionOfIndexObject, err, zap.String("path", path)) return false } @@ -165,7 +167,7 @@ func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.Req } } -func (h *Handler) byAttributeSearchMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), attr string, pathFormer func(string) string) MiddlewareFunc { +func (h *Handler) byAttributeSearchMiddleware(handler ObjectHandlerFunc, attr string, pathFormer func(string) string) MiddlewareFunc { return func(prm MiddlewareParam) bool { ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAttributeSearch") defer span.End() @@ -196,7 +198,7 @@ func (h *Handler) byAttributeSearchMiddleware(handler func(context.Context, *fas } } -func (h *Handler) byAddressMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) MiddlewareFunc { +func (h *Handler) byAddressMiddleware(handler ObjectHandlerFunc) MiddlewareFunc { return func(prm MiddlewareParam) bool { ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAddress") defer span.End() diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 59a19ed..4d1dc31 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -11,8 +11,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" @@ -173,7 +173,7 @@ type Handler struct { ownerID *user.ID config Config containerResolver ContainerResolver - tree layer.TreeService + tree *tree.Tree cache *cache.BucketCache workerPool *ants.Pool corsCnrID cid.ID @@ -190,7 +190,7 @@ type AppParams struct { CORSCache *cache.CORSCache } -func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler { +func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { return &Handler{ log: params.Logger, frostfs: params.FrostFS, @@ -205,36 +205,6 @@ func New(params *AppParams, config Config, tree layer.TreeService, workerPool *a } } -// byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that -// prepares request and object address to it. -func (h *Handler) byNativeAddress(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, objID oid.ID, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { - ctx, span := tracing.StartSpanFromContext(ctx, "handler.byNativeAddress") - defer span.End() - - addr := newAddress(cnrID, objID) - handler(ctx, req, addr) -} - -// byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that -// resolves object address from S3-like path /. -func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, bktInfo *data.BucketInfo, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { - ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path") - defer span.End() - - foundOID, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, path) - if err != nil { - h.logAndSendError(ctx, req, logs.FailedToGetLatestVersionOfObject, err, zap.String("path", path)) - return - } - if foundOID.IsDeleteMarker { - h.logAndSendError(ctx, req, logs.ObjectWasDeleted, ErrObjectNotFound) - return - } - - addr := newAddress(bktInfo.CID, foundOID.OID) - handler(ctx, req, addr) -} - // byAttribute is a wrapper similar to byNativeAddress. func (h *Handler) byAttribute(ctx context.Context, req *fasthttp.RequestCtx, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { cidParam, _ := req.UserValue("cid").(string) diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index dbb037d..622940e 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -14,9 +14,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" @@ -36,32 +37,6 @@ import ( "go.uber.org/zap/zaptest" ) -type treeServiceMock struct { - system map[string]map[string]*data.BaseNodeVersion -} - -func newTreeService() *treeServiceMock { - return &treeServiceMock{ - system: make(map[string]map[string]*data.BaseNodeVersion), - } -} - -func (t *treeServiceMock) CheckSettingsNodeExists(context.Context, *data.BucketInfo) error { - _, ok := t.system["bucket-settings"] - if !ok { - return layer.ErrNodeNotFound - } - return nil -} - -func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeInfo, string, error) { - return nil, "", nil -} - -func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*data.NodeVersion, error) { - return nil, nil -} - type configMock struct { additionalFilenameSearch bool additionalSlashSearch bool @@ -82,7 +57,7 @@ func (c *configMock) IndexPageEnabled() bool { } func (c *configMock) IndexPageTemplate() string { - return "" + return templates.DefaultIndexTemplate } func (c *configMock) IndexPageNativeTemplate() string { @@ -124,7 +99,7 @@ type handlerContext struct { h *Handler frostfs *TestFrostFS - tree *treeServiceMock + tree *treeServiceClientMock cfg *configMock } @@ -174,14 +149,14 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { }), } - treeMock := newTreeService() + treeMock := newTreeServiceClientMock() cfgMock := &configMock{} workerPool, err := ants.NewPool(1) if err != nil { return nil, err } - handler := New(params, cfgMock, treeMock, workerPool) + handler := New(params, cfgMock, tree.NewTree(treeMock, logger), workerPool) return &handlerContext{ key: key, @@ -532,6 +507,100 @@ func TestGetObjectWithFallback(t *testing.T) { }) } +func TestIndex(t *testing.T) { + ctx := middleware.SetNamespace(context.Background(), "") + + t.Run("s3", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + hc.tree.containers[cnrID.String()] = containerInfo{ + trees: map[string]map[string]nodeResponse{ + "system": {"bucket-settings": nodeResponse{nodeID: 1}}, + "version": { + "": nodeResponse{}, //root + "prefix": nodeResponse{ + nodeID: 1, + meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}}, + "obj1": nodeResponse{ + parentID: 1, + nodeID: 2, + meta: []nodeMeta{ + {key: tree.FileNameKey, value: []byte("obj1")}, + {key: "OID", value: []byte(obj1ID.String())}, + }, + }, + }, + }, + } + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.indexEnabled = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, "bucket", "dummy") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy") + }) + + t.Run("native", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.indexEnabled = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy") + }) +} + func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) diff --git a/internal/handler/head.go b/internal/handler/head.go index e6d9a30..508dc37 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -9,8 +9,8 @@ import ( "strconv" "time" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -142,7 +142,7 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { } checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) - if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) { h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) return } @@ -157,7 +157,7 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { indexPageEnabled := h.config.IndexPageEnabled() if checkS3Err == nil { - run(prm, h.errorMiddleware(logs.ObjectNotFound, layer.ErrNodeNotFound), + run(prm, h.errorMiddleware(logs.ObjectNotFound, tree.ErrNodeNotFound), Middleware{Func: h.byS3PathMiddleware(h.headObject, noopFormer), Enabled: true}, Middleware{Func: h.byS3PathMiddleware(h.headObject, indexFormer), Enabled: indexPageEnabled}, ) diff --git a/internal/handler/tree_service_client_mock_test.go b/internal/handler/tree_service_client_mock_test.go new file mode 100644 index 0000000..f3af52a --- /dev/null +++ b/internal/handler/tree_service_client_mock_test.go @@ -0,0 +1,141 @@ +package handler + +import ( + "context" + "errors" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" +) + +type nodeMeta struct { + key string + value []byte +} + +func (m nodeMeta) GetKey() string { + return m.key +} + +func (m nodeMeta) GetValue() []byte { + return m.value +} + +type nodeResponse struct { + meta []nodeMeta + nodeID uint64 + parentID uint64 + timestamp uint64 +} + +func (n nodeResponse) GetNodeID() []uint64 { + return []uint64{n.nodeID} +} + +func (n nodeResponse) GetParentID() []uint64 { + return []uint64{n.parentID} +} + +func (n nodeResponse) GetTimestamp() []uint64 { + return []uint64{n.timestamp} +} + +func (n nodeResponse) GetMeta() []tree.Meta { + res := make([]tree.Meta, len(n.meta)) + for i, value := range n.meta { + res[i] = value + } + return res +} + +type containerInfo struct { + trees map[string]map[string]nodeResponse +} + +type treeServiceClientMock struct { + containers map[string]containerInfo +} + +func newTreeServiceClientMock() *treeServiceClientMock { + return &treeServiceClientMock{ + containers: make(map[string]containerInfo), + } +} + +func (t *treeServiceClientMock) GetNodes(_ context.Context, p *tree.GetNodesParams) ([]tree.NodeResponse, error) { + cnr, ok := t.containers[p.CnrID.EncodeToString()] + if !ok { + return nil, tree.ErrNodeNotFound + } + + tr, ok := cnr.trees[p.TreeID] + if !ok { + return nil, tree.ErrNodeNotFound + } + + node, ok := tr[strings.Join(p.Path, "/")] + if !ok { + return nil, tree.ErrNodeNotFound + } + + return []tree.NodeResponse{node}, nil +} + +func (t *treeServiceClientMock) GetSubTree(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, _ bool) ([]tree.NodeResponse, error) { + cnr, ok := t.containers[bktInfo.CID.EncodeToString()] + if !ok { + return nil, tree.ErrNodeNotFound + } + + tr, ok := cnr.trees[treeID] + if !ok { + return nil, tree.ErrNodeNotFound + } + + if len(rootID) != 1 { + return nil, errors.New("invalid rootID") + } + + var root *nodeResponse + for _, v := range tr { + if v.nodeID == rootID[0] { + root = &v + break + } + } + + if root == nil { + return nil, tree.ErrNodeNotFound + } + + var res []nodeResponse + if depth == 0 { + for _, v := range tr { + res = append(res, v) + } + } else { + res = append(res, *root) + depthIndex := 0 + for i := uint32(0); i < depth-1; i++ { + childrenCount := 0 + for _, v := range tr { + for j := range res[depthIndex:] { + if v.parentID == res[j].nodeID { + res = append(res, v) + childrenCount++ + break + } + } + } + depthIndex = len(res) - childrenCount + } + } + + res2 := make([]tree.NodeResponse, len(res)) + for i := range res { + res2[i] = res[i] + } + + return res2, nil +} diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 8cb070d..c17b878 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -6,9 +6,9 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -93,7 +93,7 @@ func formErrorResponse(err error) (string, int) { switch { case errors.Is(err, ErrAccessDenied): return fmt.Sprintf("Storage Access Denied:\n%v", err), fasthttp.StatusForbidden - case errors.Is(err, layer.ErrNodeAccessDenied): + case errors.Is(err, tree.ErrNodeAccessDenied): return fmt.Sprintf("Tree Access Denied:\n%v", err), fasthttp.StatusForbidden case errors.Is(err, ErrQuotaLimitReached): return fmt.Sprintf("Quota Reached:\n%v", err), fasthttp.StatusConflict @@ -101,7 +101,7 @@ func formErrorResponse(err error) (string, int) { return fmt.Sprintf("Container Not Found:\n%v", err), fasthttp.StatusNotFound case errors.Is(err, ErrObjectNotFound): return fmt.Sprintf("Object Not Found:\n%v", err), fasthttp.StatusNotFound - case errors.Is(err, layer.ErrNodeNotFound): + case errors.Is(err, tree.ErrNodeNotFound): return fmt.Sprintf("Tree Node Not Found:\n%v", err), fasthttp.StatusNotFound case errors.Is(err, ErrGatewayTimeout): return fmt.Sprintf("Gateway Timeout:\n%v", err), fasthttp.StatusGatewayTimeout diff --git a/internal/layer/tree_service.go b/internal/layer/tree_service.go deleted file mode 100644 index ff80543..0000000 --- a/internal/layer/tree_service.go +++ /dev/null @@ -1,24 +0,0 @@ -package layer - -import ( - "context" - "errors" - - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" -) - -// TreeService provide interface to interact with tree service using s3 data models. -type TreeService interface { - GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) - GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) - CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error -} - -var ( - // ErrNodeNotFound is returned from Tree service in case of not found error. - ErrNodeNotFound = errors.New("not found") - - // ErrNodeAccessDenied is returned from Tree service in case of access denied error. - ErrNodeAccessDenied = errors.New("access denied") -) diff --git a/internal/templates/index.gotmpl b/internal/templates/index.gotmpl index b14cc06..4c03404 100644 --- a/internal/templates/index.gotmpl +++ b/internal/templates/index.gotmpl @@ -1,11 +1,9 @@ {{$container := .Container}} -{{ $prefix := trimPrefix .Prefix }} - Index of {{.Protocol}}://{{$container}} - /{{if $prefix}}/{{$prefix}}/{{end}} + Index of {{.Protocol}}://{{$container}}/{{.Prefix}} -

Index of {{.Protocol}}://{{$container}}/{{if $prefix}}{{$prefix}}/{{end}}

+

Index of {{.Protocol}}://{{$container}}/{{.Prefix}}

{{ if .HasErrors }}
Errors occurred while processing the request. Perhaps some objects are missing @@ -57,11 +55,11 @@
- ⮐.. + ⮐..