vendor: update all dependencies

This commit is contained in:
Nick Craig-Wood 2019-08-26 18:00:17 +01:00
parent d1a39dcc4b
commit af192d2507
232 changed files with 15744 additions and 1710 deletions

View file

@ -27,7 +27,6 @@ Making a release
Early in the next release cycle update the vendored dependencies
* Review any pinned packages in go.mod and remove if possible
* GO111MODULE=on go get -u github.com/spf13/cobra@master
* make update
* git status
* git add new files

58
go.mod
View file

@ -2,68 +2,68 @@ module github.com/rclone/rclone
require (
bazil.org/fuse v0.0.0-20180421153158-65cc252bf669
cloud.google.com/go v0.40.0 // indirect
github.com/Azure/azure-pipeline-go v0.1.9
github.com/Azure/azure-storage-blob-go v0.6.0
cloud.google.com/go v0.44.3 // indirect
github.com/Azure/azure-pipeline-go v0.2.2
github.com/Azure/azure-storage-blob-go v0.8.0
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78
github.com/Azure/go-autorest/autorest/adal v0.6.0 // indirect
github.com/Unknwon/goconfig v0.0.0-20190425194916-3dba17dd7b9e
github.com/a8m/tree v0.0.0-20181222104329-6a0b80129de4
github.com/abbot/go-http-auth v0.4.0
github.com/anacrolix/dms v1.0.0
github.com/atotto/clipboard v0.1.2
github.com/aws/aws-sdk-go v1.20.10
github.com/aws/aws-sdk-go v1.23.8
github.com/billziss-gh/cgofuse v1.1.0
github.com/coreos/bbolt v1.3.3
github.com/djherbis/times v1.2.0
github.com/dropbox/dropbox-sdk-go-unofficial v5.4.0+incompatible
github.com/goftp/file-driver v0.0.0-20180502053751-5d604a0fc0c9 // indirect
github.com/goftp/server v0.0.0-20190304020633-eabccc535b5a
github.com/goftp/server v0.0.0-20190712054601-1149070ae46b
github.com/google/go-cmp v0.3.1 // indirect
github.com/google/go-querystring v1.0.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/jlaffaye/ftp v0.0.0-20190721194432-7cd8b0bcf3fc
github.com/jzelinskie/whirlpool v0.0.0-20170603002051-c19460b8caa6
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/koofr/go-httpclient v0.0.0-20180104120329-03786175608a
github.com/koofr/go-koofrclient v0.0.0-20190715104037-2c5813f3c29c
github.com/kr/fs v0.1.0 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/koofr/go-httpclient v0.0.0-20190818202018-e0dc8fd921dc
github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a
github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
github.com/mattn/go-runewidth v0.0.4
github.com/mitchellh/go-homedir v1.1.0
github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2
github.com/ncw/swift v1.0.47
github.com/nsf/termbox-go v0.0.0-20190624072549-eeb6cd0a1762
github.com/ncw/swift v1.0.49
github.com/nsf/termbox-go v0.0.0-20190817171036-93860e161317
github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd
github.com/onsi/ginkgo v1.8.0 // indirect
github.com/onsi/gomega v1.5.0 // indirect
github.com/onsi/ginkgo v1.9.0 // indirect
github.com/onsi/gomega v1.6.0 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.8.1
github.com/pkg/sftp v1.10.1-0.20190523025818-e98a7bef6829
github.com/putdotio/go-putio v0.0.0-20190731220109-37c0795af843
github.com/pkg/sftp v1.10.1
github.com/putdotio/go-putio v0.0.0-20190822121956-19b9c636c877
github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46
github.com/sevlyar/go-daemon v0.1.5
github.com/sirupsen/logrus v1.4.2
github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e
github.com/smartystreets/assertions v1.0.0 // indirect
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
github.com/smartystreets/assertions v1.0.1 // indirect
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.3
github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709
github.com/stretchr/testify v1.4.0
github.com/t3rm1n4l/go-mega v0.0.0-20190528125457-55e675378686
github.com/xanzy/ssh-agent v0.2.1
github.com/yunify/qingstor-sdk-go/v3 v3.0.2
go.etcd.io/bbolt v1.3.3 // indirect
go.opencensus.io v0.22.0 // indirect
golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4
golang.org/x/net v0.0.0-20190620200207-3b0461eec859
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb
golang.org/x/sys v0.0.0-20190826163724-acd9dae8e8cc
golang.org/x/text v0.3.2
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
google.golang.org/api v0.7.0
google.golang.org/appengine v1.6.1 // indirect
google.golang.org/genproto v0.0.0-20190626174449-989357319d63 // indirect
google.golang.org/grpc v1.21.1 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
google.golang.org/api v0.9.0
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect
google.golang.org/grpc v1.23.0 // indirect
gopkg.in/yaml.v2 v2.2.2
)

143
go.sum
View file

@ -3,16 +3,35 @@ bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxo
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.40.0 h1:FjSY7bOj+WzJe6TZRVtXI2b9kAYvtNg4lMbcH2+MUkk=
cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
github.com/Azure/azure-pipeline-go v0.1.9 h1:u7JFb9fFTE6Y/j8ae2VK33ePrRqJqoCM/IWkQdAZ+rg=
github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
github.com/Azure/azure-storage-blob-go v0.6.0 h1:SEATKb3LIHcaSIX+E6/K4kJpwfuozFEsmt5rS56N6CE=
github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
cloud.google.com/go v0.44.3 h1:0sMegbmn/8uTwpNkB0q9cLEpZ2W5a6kl+wtBQgPWBJQ=
cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY=
github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs=
github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
github.com/Azure/go-autorest/autorest/adal v0.6.0 h1:UCTq22yE3RPgbU/8u4scfnnzuCW6pwQ9n+uBtV78ouo=
github.com/Azure/go-autorest/autorest/adal v0.6.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM=
github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc=
github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY=
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
github.com/Unknwon/goconfig v0.0.0-20190425194916-3dba17dd7b9e h1:ZaFHdRwv6wJQMYsg5qITIsqWRqZRvUETiq0xxrl+8fc=
github.com/Unknwon/goconfig v0.0.0-20190425194916-3dba17dd7b9e/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
@ -30,8 +49,8 @@ github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/atotto/clipboard v0.1.2 h1:YZCtFu5Ie8qX2VmVTBnrqLSiU9XOWwqNRmdT3gIQzbY=
github.com/atotto/clipboard v0.1.2/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
github.com/aws/aws-sdk-go v1.20.10 h1:wrwdNYb9Xe1ahNEEXl0M/O/g/dmonWag1TFAKVNQulM=
github.com/aws/aws-sdk-go v1.20.10/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.23.8 h1:G/azJoBN0pnhB3B+0eeC4yyVFYIIad6bbzg6wwtImqk=
github.com/aws/aws-sdk-go v1.23.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/billziss-gh/cgofuse v1.1.0 h1:tATn9ZDvuPcOVlvR4tJitGHgAqy1y18+4mKmRfdfjec=
github.com/billziss-gh/cgofuse v1.1.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
@ -47,6 +66,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/djherbis/times v1.2.0 h1:xANXjsC/iBqbO00vkWlYwPWgBgEVU6m6AFYg0Pic+Mc=
github.com/djherbis/times v1.2.0/go.mod h1:CGMZlo255K5r4Yw0b9RRfFQpM2y7uOmxg4jm9HsaVf8=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
@ -59,41 +80,52 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod
github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/goftp/file-driver v0.0.0-20180502053751-5d604a0fc0c9 h1:cC0Hbb+18DJ4i6ybqDybvj4wdIDS4vnD0QEci98PgM8=
github.com/goftp/file-driver v0.0.0-20180502053751-5d604a0fc0c9/go.mod h1:GpOj6zuVBG3Inr9qjEnuVTgBlk2lZ1S9DcoFiXWyKss=
github.com/goftp/server v0.0.0-20190304020633-eabccc535b5a h1:XTJuuzIub3zu2FgPqdFM9XFYYisXWu2hN/rFwayAIcY=
github.com/goftp/server v0.0.0-20190304020633-eabccc535b5a/go.mod h1:k/SS6VWkxY7dHPhoMQ8IdRu8L4lQtmGbhyXGg+vCnXE=
github.com/goftp/server v0.0.0-20190712054601-1149070ae46b h1:2rRhW1AEs/240C6fpmgGFKlTnh/339r2Cg+ahrkSodo=
github.com/goftp/server v0.0.0-20190712054601-1149070ae46b/go.mod h1:k/SS6VWkxY7dHPhoMQ8IdRu8L4lQtmGbhyXGg+vCnXE=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4=
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk=
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf h1:2IYBd5TD/maMqTU2YUzp2tJL4cNaOYQ9EBullN9t9pk=
github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
github.com/jlaffaye/ftp v0.0.0-20190721194432-7cd8b0bcf3fc h1:Mc2Gk3kF0Uqx+cI97pN0gbgZb0DVW2L+htrZSKkOmtE=
github.com/jlaffaye/ftp v0.0.0-20190721194432-7cd8b0bcf3fc/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@ -106,10 +138,12 @@ github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uia
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/koofr/go-httpclient v0.0.0-20180104120329-03786175608a h1:W+gnfphB7WpRj0rbTF40e3edULfri4fou2kUFw6AF3A=
github.com/koofr/go-httpclient v0.0.0-20180104120329-03786175608a/go.mod h1:3xszwh+rNrYk1r9SStc4iJ326gne1OaBcrdB1ACsbzI=
github.com/koofr/go-koofrclient v0.0.0-20190715104037-2c5813f3c29c h1:MaVra5jrE7MxT4k0v0JD2Gh6h94BFu2DgDYq+zMGBW0=
github.com/koofr/go-koofrclient v0.0.0-20190715104037-2c5813f3c29c/go.mod h1:MRAz4Gsxd+OzrZ0owwrUHc0zLESL+1Y5syqK/sJxK2A=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/koofr/go-httpclient v0.0.0-20190818202018-e0dc8fd921dc h1:Y+Dob6xos1WxridFN8sIiWPOlEQzgeF9ij9dXOsXK44=
github.com/koofr/go-httpclient v0.0.0-20190818202018-e0dc8fd921dc/go.mod h1:3xszwh+rNrYk1r9SStc4iJ326gne1OaBcrdB1ACsbzI=
github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a h1:02cx9xF4W2FQ1oh8CK9dWV5BnZK2mUtcbr9xR+bZiKk=
github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a/go.mod h1:MRAz4Gsxd+OzrZ0owwrUHc0zLESL+1Y5syqK/sJxK2A=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@ -119,6 +153,10 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-gtk v0.0.0-20190405072524-4deadb416788/go.mod h1:PwzwfeB5syFHXORC3MtPylVcjIoTDT/9cvkKpEndGVI=
github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb h1:hXqqXzQtJbENrsb+rsIqkVqcg4FUJL0SQFGw08Dgivw=
github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
github.com/mattn/go-pointer v0.0.0-20180825124634-49522c3f3791/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
@ -128,17 +166,17 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2 h1:VlXvEx6JbFp7F9iz92zXP2Ew+9VupSpfybr+TxmjdH0=
github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2/go.mod h1:MLIrzg7gp/kzVBxRE1olT7CWYMCklcUWU+ekoxOD9x0=
github.com/ncw/swift v1.0.47 h1:4DQRPj35Y41WogBxyhOXlrI37nzGlyEcsforeudyYPQ=
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
github.com/nsf/termbox-go v0.0.0-20190624072549-eeb6cd0a1762 h1:44Lv0bNi88GweB54TCjB/lEJgp+2Ze5WFpwNu0nh0ag=
github.com/nsf/termbox-go v0.0.0-20190624072549-eeb6cd0a1762/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ=
github.com/ncw/swift v1.0.49 h1:eQaKIjSt/PXLKfYgzg01nevmO+CMXfXGRhB1gOhDs7E=
github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
github.com/nsf/termbox-go v0.0.0-20190817171036-93860e161317 h1:hhGN4SFXgXo61Q4Sjj/X9sBjyeSa2kdpaOzCO+8EVQw=
github.com/nsf/termbox-go v0.0.0-20190817171036-93860e161317/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ=
github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd h1:+iAPaTbi1gZpcpDwe/BW1fx7Xoesv69hLNGPheoyhBs=
github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd/go.mod h1:4soZNh0zW0LtYGdQ416i0jO0EIqMGcbtaspRS4BDvRQ=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc=
github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M=
github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@ -147,12 +185,12 @@ github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14/go.mod h1:jVbl
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.10.1-0.20190523025818-e98a7bef6829 h1:I+1BDgqX1nXLUL5Uio219Nj0Tne+xW0gp2EOENDk00M=
github.com/pkg/sftp v1.10.1-0.20190523025818-e98a7bef6829/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk=
github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc=
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/putdotio/go-putio v0.0.0-20190731220109-37c0795af843 h1:Hf9Zw1N+0gQolpRwWOwisCombJ/9anLxUwFkLMQL22c=
github.com/putdotio/go-putio v0.0.0-20190731220109-37c0795af843/go.mod h1:EWtDL88jJLLWZzywr0QaPO+mGP8gFpvl8dcox8qTk3Y=
github.com/putdotio/go-putio v0.0.0-20190822121956-19b9c636c877 h1:sKIa5MAIViLAnQbEo+uiDi2FMowy8KcdZW8XZpmyNxs=
github.com/putdotio/go-putio v0.0.0-20190822121956-19b9c636c877/go.mod h1:EWtDL88jJLLWZzywr0QaPO+mGP8gFpvl8dcox8qTk3Y=
github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46 h1:w2CpS5muK+jyydnmlkqpAhzKmHmMBzBkfYUDjQNS1Dk=
github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46/go.mod h1:U2bmx0hDj8EyDdcxmD5t3XHDnBFnyNNc22n1R4008eM=
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
@ -165,11 +203,11 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e h1:VAzdS5Nw68fbf5RZ8RDVlUvPXNU6Z3jtPCK/qvm4FoQ=
github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
@ -186,6 +224,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709 h1:Ko2LQMrRU+Oy/+EDBwX7eZ2jp3C47eDBB8EIhKTun+I=
github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/t3rm1n4l/go-mega v0.0.0-20190528125457-55e675378686 h1:U7mF+tjDK9zWoxCU+kBNa1XT7WZMF5bjwtRpjeIkSYw=
github.com/t3rm1n4l/go-mega v0.0.0-20190528125457-55e675378686/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0=
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@ -206,14 +246,17 @@ golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 h1:ydJNl0ENAG67pFbB+9tfhiL2pYqLhfoaZFw/cjLhY4A=
golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -227,6 +270,8 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
@ -242,14 +287,16 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190415145633-3fd5a3612ccd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826163724-acd9dae8e8cc h1:Cgiu447JccQnHt7K/DbJbw1DbXAUHwOtU7ObeOCVsc4=
golang.org/x/sys v0.0.0-20190826163724-acd9dae8e8cc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@ -261,14 +308,20 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
google.golang.org/api v0.7.0 h1:9sdfJOzWlkqPltHAuzT2Cp+yrBeY1KRVYgms8soxMwM=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.9.0 h1:jbyannxz0XFD3zdjgrSUsaJbgpH4eTrkdhRChkHPfO8=
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -279,15 +332,16 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
google.golang.org/genproto v0.0.0-20190626174449-989357319d63 h1:UsSJe9fhWNSz6emfIGPpH5DF23t7ALo2Pf3sC+/hsdg=
google.golang.org/genproto v0.0.0-20190626174449-989357319d63/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
@ -300,4 +354,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

View file

@ -2,6 +2,7 @@ package pipeline
import (
"context"
"github.com/mattn/go-ieproxy"
"net"
"net/http"
"os"
@ -204,7 +205,7 @@ func newDefaultHTTPClient() *http.Client {
// We want the Transport to have a large connection pool
return &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
Proxy: ieproxy.GetProxyFunc(),
// We use Dial instead of DialContext as DialContext has been reported to cause slower performance.
Dial /*Context*/ : (&net.Dialer{
Timeout: 30 * time.Second,
@ -253,3 +254,31 @@ type methodFactoryMarker struct {
func (methodFactoryMarker) New(next Policy, po *PolicyOptions) Policy {
panic("methodFactoryMarker policy should have been replaced with a method policy")
}
// LogSanitizer can be implemented to clean secrets from lines logged by ForceLog
// By default no implemetation is provided here, because pipeline may be used in many different
// contexts, so the correct implementation is context-dependent
type LogSanitizer interface {
SanitizeLogMessage(raw string) string
}
var sanitizer LogSanitizer
var enableForceLog bool = true
// SetLogSanitizer can be called to supply a custom LogSanitizer.
// There is no threadsafety or locking on the underlying variable,
// so call this function just once at startup of your application
// (Don't later try to change the sanitizer on the fly).
func SetLogSanitizer(s LogSanitizer)(){
sanitizer = s
}
// SetForceLogEnabled can be used to disable ForceLog
// There is no threadsafety or locking on the underlying variable,
// so call this function just once at startup of your application
// (Don't later try to change the setting on the fly).
func SetForceLogEnabled(enable bool)() {
enableForceLog = enable
}

View file

@ -0,0 +1,14 @@
package pipeline
// ForceLog should rarely be used. It forceable logs an entry to the
// Windows Event Log (on Windows) or to the SysLog (on Linux)
func ForceLog(level LogLevel, msg string) {
if !enableForceLog {
return
}
if sanitizer != nil {
msg = sanitizer.SanitizeLogMessage(msg)
}
forceLog(level, msg)
}

View file

@ -7,9 +7,9 @@ import (
"log/syslog"
)
// ForceLog should rarely be used. It forceable logs an entry to the
// forceLog should rarely be used. It forceable logs an entry to the
// Windows Event Log (on Windows) or to the SysLog (on Linux)
func ForceLog(level LogLevel, msg string) {
func forceLog(level LogLevel, msg string) {
if defaultLogger == nil {
return // Return fast if we failed to create the logger.
}

View file

@ -6,9 +6,9 @@ import (
"unsafe"
)
// ForceLog should rarely be used. It forceable logs an entry to the
// forceLog should rarely be used. It forceable logs an entry to the
// Windows Event Log (on Windows) or to the SysLog (on Linux)
func ForceLog(level LogLevel, msg string) {
func forceLog(level LogLevel, msg string) {
var el eventType
switch level {
case LogError, LogFatal, LogPanic:

View file

@ -5,5 +5,5 @@ const (
UserAgent = "azure-pipeline-go/" + Version
// Version is the semantic version (see http://semver.org) of the pipeline package.
Version = "0.1.0"
Version = "0.2.1"
)

File diff suppressed because it is too large Load diff

View file

@ -66,7 +66,7 @@ func UploadBufferToBlockBlob(ctx context.Context, b []byte,
if o.BlockSize == 0 {
// If bufferSize > (BlockBlobMaxStageBlockBytes * BlockBlobMaxBlocks), then error
if bufferSize > BlockBlobMaxStageBlockBytes*BlockBlobMaxBlocks {
return nil, errors.New("Buffer is too large to upload to a block blob")
return nil, errors.New("buffer is too large to upload to a block blob")
}
// If bufferSize <= BlockBlobMaxUploadBlobBytes, then Upload should be used with just 1 I/O request
if bufferSize <= BlockBlobMaxUploadBlobBytes {
@ -76,7 +76,7 @@ func UploadBufferToBlockBlob(ctx context.Context, b []byte,
if o.BlockSize < BlobDefaultDownloadBlockSize { // If the block size is smaller than 4MB, round up to 4MB
o.BlockSize = BlobDefaultDownloadBlockSize
}
// StageBlock will be called with blockSize blocks and a parallelism of (BufferSize / BlockSize).
// StageBlock will be called with blockSize blocks and a Parallelism of (BufferSize / BlockSize).
}
}
@ -95,12 +95,12 @@ func UploadBufferToBlockBlob(ctx context.Context, b []byte,
progress := int64(0)
progressLock := &sync.Mutex{}
err := doBatchTransfer(ctx, batchTransferOptions{
operationName: "UploadBufferToBlockBlob",
transferSize: bufferSize,
chunkSize: o.BlockSize,
parallelism: o.Parallelism,
operation: func(offset int64, count int64) error {
err := DoBatchTransfer(ctx, BatchTransferOptions{
OperationName: "UploadBufferToBlockBlob",
TransferSize: bufferSize,
ChunkSize: o.BlockSize,
Parallelism: o.Parallelism,
Operation: func(offset int64, count int64, ctx context.Context) error {
// This function is called once per block.
// It is passed this block's offset within the buffer and its count of bytes
// Prepare to read the proper block/section of the buffer
@ -198,13 +198,16 @@ func downloadBlobToBuffer(ctx context.Context, blobURL BlobURL, offset int64, co
progress := int64(0)
progressLock := &sync.Mutex{}
err := doBatchTransfer(ctx, batchTransferOptions{
operationName: "downloadBlobToBuffer",
transferSize: count,
chunkSize: o.BlockSize,
parallelism: o.Parallelism,
operation: func(chunkStart int64, count int64) error {
err := DoBatchTransfer(ctx, BatchTransferOptions{
OperationName: "downloadBlobToBuffer",
TransferSize: count,
ChunkSize: o.BlockSize,
Parallelism: o.Parallelism,
Operation: func(chunkStart int64, count int64, ctx context.Context) error {
dr, err := blobURL.Download(ctx, chunkStart+offset, count, o.AccessConditions, false)
if err != nil {
return err
}
body := dr.Body(o.RetryReaderOptionsPerBlock)
if o.Progress != nil {
rangeProgress := int64(0)
@ -282,64 +285,69 @@ func DownloadBlobToFile(ctx context.Context, blobURL BlobURL, offset int64, coun
///////////////////////////////////////////////////////////////////////////////
// BatchTransferOptions identifies options used by doBatchTransfer.
type batchTransferOptions struct {
transferSize int64
chunkSize int64
parallelism uint16
operation func(offset int64, chunkSize int64) error
operationName string
// BatchTransferOptions identifies options used by DoBatchTransfer.
type BatchTransferOptions struct {
TransferSize int64
ChunkSize int64
Parallelism uint16
Operation func(offset int64, chunkSize int64, ctx context.Context) error
OperationName string
}
// doBatchTransfer helps to execute operations in a batch manner.
func doBatchTransfer(ctx context.Context, o batchTransferOptions) error {
// DoBatchTransfer helps to execute operations in a batch manner.
// Can be used by users to customize batch works (for other scenarios that the SDK does not provide)
func DoBatchTransfer(ctx context.Context, o BatchTransferOptions) error {
if o.ChunkSize == 0 {
return errors.New("ChunkSize cannot be 0")
}
// Prepare and do parallel operations.
numChunks := uint16(((o.transferSize - 1) / o.chunkSize) + 1)
operationChannel := make(chan func() error, o.parallelism) // Create the channel that release 'parallelism' goroutines concurrently
numChunks := uint16(((o.TransferSize - 1) / o.ChunkSize) + 1)
operationChannel := make(chan func() error, o.Parallelism) // Create the channel that release 'Parallelism' goroutines concurrently
operationResponseChannel := make(chan error, numChunks) // Holds each response
ctx, cancel := context.WithCancel(ctx)
defer cancel()
// Create the goroutines that process each operation (in parallel).
if o.parallelism == 0 {
o.parallelism = 5 // default parallelism
if o.Parallelism == 0 {
o.Parallelism = 5 // default Parallelism
}
for g := uint16(0); g < o.parallelism; g++ {
for g := uint16(0); g < o.Parallelism; g++ {
//grIndex := g
go func() {
for f := range operationChannel {
//fmt.Printf("[%s] gr-%d start action\n", o.operationName, grIndex)
err := f()
operationResponseChannel <- err
//fmt.Printf("[%s] gr-%d end action\n", o.operationName, grIndex)
}
}()
}
// Add each chunk's operation to the channel.
for chunkNum := uint16(0); chunkNum < numChunks; chunkNum++ {
curChunkSize := o.chunkSize
curChunkSize := o.ChunkSize
if chunkNum == numChunks-1 { // Last chunk
curChunkSize = o.transferSize - (int64(chunkNum) * o.chunkSize) // Remove size of all transferred chunks from total
curChunkSize = o.TransferSize - (int64(chunkNum) * o.ChunkSize) // Remove size of all transferred chunks from total
}
offset := int64(chunkNum) * o.chunkSize
offset := int64(chunkNum) * o.ChunkSize
operationChannel <- func() error {
return o.operation(offset, curChunkSize)
return o.Operation(offset, curChunkSize, ctx)
}
}
close(operationChannel)
// Wait for the operations to complete.
var firstErr error = nil
for chunkNum := uint16(0); chunkNum < numChunks; chunkNum++ {
responseError := <-operationResponseChannel
if responseError != nil {
cancel() // As soon as any operation fails, cancel all remaining operation calls
return responseError // No need to process anymore responses
// record the first error (the original error which should cause the other chunks to fail with canceled context)
if responseError != nil && firstErr == nil {
cancel() // As soon as any operation fails, cancel all remaining operation calls
firstErr = responseError
}
}
return nil
return firstErr
}
////////////////////////////////////////////////////////////////////////////////////////////////

View file

@ -124,6 +124,11 @@ func (up BlobURLParts) URL() url.URL {
rawQuery := up.UnparsedParams
//If no snapshot is initially provided, fill it in from the SAS query properties to help the user
if up.Snapshot == "" && !up.SAS.snapshotTime.IsZero() {
up.Snapshot = up.SAS.snapshotTime.Format(SnapshotTimeFormat)
}
// Concatenate blob snapshot query parameter (if it exists)
if up.Snapshot != "" {
if len(rawQuery) > 0 {

View file

@ -14,9 +14,10 @@ type BlobSASSignatureValues struct {
Protocol SASProtocol `param:"spr"` // See the SASProtocol* constants
StartTime time.Time `param:"st"` // Not specified if IsZero
ExpiryTime time.Time `param:"se"` // Not specified if IsZero
Permissions string `param:"sp"` // Create by initializing a ContainerSASPermissions or BlobSASPermissions and then call String()
IPRange IPRange `param:"sip"`
Identifier string `param:"si"`
SnapshotTime time.Time
Permissions string `param:"sp"` // Create by initializing a ContainerSASPermissions or BlobSASPermissions and then call String()
IPRange IPRange `param:"sip"`
Identifier string `param:"si"`
ContainerName string
BlobName string // Use "" to create a Container SAS
CacheControl string // rscc
@ -26,11 +27,24 @@ type BlobSASSignatureValues struct {
ContentType string // rsct
}
// NewSASQueryParameters uses an account's shared key credential to sign this signature values to produce
// NewSASQueryParameters uses an account's StorageAccountCredential to sign this signature values to produce
// the proper SAS query parameters.
func (v BlobSASSignatureValues) NewSASQueryParameters(sharedKeyCredential *SharedKeyCredential) (SASQueryParameters, error) {
// See: StorageAccountCredential. Compatible with both UserDelegationCredential and SharedKeyCredential
func (v BlobSASSignatureValues) NewSASQueryParameters(credential StorageAccountCredential) (SASQueryParameters, error) {
resource := "c"
if v.BlobName == "" {
if credential == nil {
return SASQueryParameters{}, fmt.Errorf("cannot sign SAS query without StorageAccountCredential")
}
if !v.SnapshotTime.IsZero() {
resource = "bs"
//Make sure the permission characters are in the correct order
perms := &BlobSASPermissions{}
if err := perms.Parse(v.Permissions); err != nil {
return SASQueryParameters{}, err
}
v.Permissions = perms.String()
} else if v.BlobName == "" {
// Make sure the permission characters are in the correct order
perms := &ContainerSASPermissions{}
if err := perms.Parse(v.Permissions); err != nil {
@ -49,27 +63,47 @@ func (v BlobSASSignatureValues) NewSASQueryParameters(sharedKeyCredential *Share
if v.Version == "" {
v.Version = SASVersion
}
startTime, expiryTime := FormatTimesForSASSigning(v.StartTime, v.ExpiryTime)
startTime, expiryTime, snapshotTime := FormatTimesForSASSigning(v.StartTime, v.ExpiryTime, v.SnapshotTime)
signedIdentifier := v.Identifier
udk := credential.getUDKParams()
if udk != nil {
udkStart, udkExpiry, _ := FormatTimesForSASSigning(udk.SignedStart, udk.SignedExpiry, time.Time{})
//I don't like this answer to combining the functions
//But because signedIdentifier and the user delegation key strings share a place, this is an _OK_ way to do it.
signedIdentifier = strings.Join([]string{
udk.SignedOid,
udk.SignedTid,
udkStart,
udkExpiry,
udk.SignedService,
udk.SignedVersion,
}, "\n")
}
// String to sign: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
stringToSign := strings.Join([]string{
v.Permissions,
startTime,
expiryTime,
getCanonicalName(sharedKeyCredential.AccountName(), v.ContainerName, v.BlobName),
v.Identifier,
getCanonicalName(credential.AccountName(), v.ContainerName, v.BlobName),
signedIdentifier,
v.IPRange.String(),
string(v.Protocol),
v.Version,
resource,
"", // signed timestamp, @TODO add for snapshot sas feature
snapshotTime, // signed timestamp
v.CacheControl, // rscc
v.ContentDisposition, // rscd
v.ContentEncoding, // rsce
v.ContentLanguage, // rscl
v.ContentType}, // rsct
"\n")
signature := sharedKeyCredential.ComputeHMACSHA256(stringToSign)
signature := ""
signature = credential.ComputeHMACSHA256(stringToSign)
p := SASQueryParameters{
// Common SAS parameters
@ -88,10 +122,22 @@ func (v BlobSASSignatureValues) NewSASQueryParameters(sharedKeyCredential *Share
contentEncoding: v.ContentEncoding,
contentLanguage: v.ContentLanguage,
contentType: v.ContentType,
snapshotTime: v.SnapshotTime,
// Calculated SAS signature
signature: signature,
}
//User delegation SAS specific parameters
if udk != nil {
p.signedOid = udk.SignedOid
p.signedTid = udk.SignedTid
p.signedStart = udk.SignedStart
p.signedExpiry = udk.SignedExpiry
p.signedService = udk.SignedService
p.signedVersion = udk.SignedVersion
}
return p, nil
}

View file

@ -0,0 +1,8 @@
package azblob
// StorageAccountCredential is a wrapper interface for SharedKeyCredential and UserDelegationCredential
type StorageAccountCredential interface {
AccountName() string
ComputeHMACSHA256(message string) (base64String string)
getUDKParams() *UserDelegationKey
}

View file

@ -71,13 +71,14 @@ func (ab AppendBlobURL) AppendBlock(ctx context.Context, body io.ReadSeeker, ac
// AppendBlockFromURL copies a new block of data from source URL to the end of the existing append blob.
// For more information, see https://docs.microsoft.com/rest/api/storageservices/append-block-from-url.
func (ab AppendBlobURL) AppendBlockFromURL(ctx context.Context, sourceURL url.URL, offset int64, count int64, ac AppendBlobAccessConditions, transactionalMD5 []byte) (*AppendBlobAppendBlockFromURLResponse, error) {
ifModifiedSince, ifUnmodifiedSince, ifMatchETag, ifNoneMatchETag := ac.ModifiedAccessConditions.pointers()
ifAppendPositionEqual, ifMaxSizeLessThanOrEqual := ac.AppendPositionAccessConditions.pointers()
func (ab AppendBlobURL) AppendBlockFromURL(ctx context.Context, sourceURL url.URL, offset int64, count int64, destinationAccessConditions AppendBlobAccessConditions, sourceAccessConditions ModifiedAccessConditions, transactionalMD5 []byte) (*AppendBlobAppendBlockFromURLResponse, error) {
ifModifiedSince, ifUnmodifiedSince, ifMatchETag, ifNoneMatchETag := destinationAccessConditions.ModifiedAccessConditions.pointers()
sourceIfModifiedSince, sourceIfUnmodifiedSince, sourceIfMatchETag, sourceIfNoneMatchETag := sourceAccessConditions.pointers()
ifAppendPositionEqual, ifMaxSizeLessThanOrEqual := destinationAccessConditions.AppendPositionAccessConditions.pointers()
return ab.abClient.AppendBlockFromURL(ctx, sourceURL.String(), 0, httpRange{offset: offset, count: count}.pointers(),
transactionalMD5, nil, transactionalMD5, ac.LeaseAccessConditions.pointers(),
transactionalMD5, nil, destinationAccessConditions.LeaseAccessConditions.pointers(),
ifMaxSizeLessThanOrEqual, ifAppendPositionEqual,
ifModifiedSince, ifUnmodifiedSince, ifMatchETag, ifNoneMatchETag, nil)
ifModifiedSince, ifUnmodifiedSince, ifMatchETag, ifNoneMatchETag, sourceIfModifiedSince, sourceIfUnmodifiedSince, sourceIfMatchETag, sourceIfNoneMatchETag, nil)
}
type AppendBlobAccessConditions struct {

View file

@ -82,8 +82,9 @@ func (bb BlockBlobURL) StageBlock(ctx context.Context, base64BlockID string, bod
// StageBlockFromURL copies the specified block from a source URL to the block blob's "staging area" to be later committed by a call to CommitBlockList.
// If count is CountToEnd (0), then data is read from specified offset to the end.
// For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-from-url.
func (bb BlockBlobURL) StageBlockFromURL(ctx context.Context, base64BlockID string, sourceURL url.URL, offset int64, count int64, ac LeaseAccessConditions) (*BlockBlobStageBlockFromURLResponse, error) {
return bb.bbClient.StageBlockFromURL(ctx, base64BlockID, 0, sourceURL.String(), httpRange{offset: offset, count: count}.pointers(), nil, nil, ac.pointers(), nil)
func (bb BlockBlobURL) StageBlockFromURL(ctx context.Context, base64BlockID string, sourceURL url.URL, offset int64, count int64, destinationAccessConditions LeaseAccessConditions, sourceAccessConditions ModifiedAccessConditions) (*BlockBlobStageBlockFromURLResponse, error) {
sourceIfModifiedSince, sourceIfUnmodifiedSince, sourceIfMatchETag, sourceIfNoneMatchETag := sourceAccessConditions.pointers()
return bb.bbClient.StageBlockFromURL(ctx, base64BlockID, 0, sourceURL.String(), httpRange{offset: offset, count: count}.pointers(), nil, nil, destinationAccessConditions.pointers(), sourceIfModifiedSince, sourceIfUnmodifiedSince, sourceIfMatchETag, sourceIfNoneMatchETag, nil)
}
// CommitBlockList writes a blob by specifying the list of block IDs that make up the blob.

View file

@ -77,13 +77,14 @@ func (pb PageBlobURL) UploadPages(ctx context.Context, offset int64, body io.Rea
// The destOffset specifies the start offset of data in page blob will be written to.
// The count must be a multiple of 512 bytes.
// For more information, see https://docs.microsoft.com/rest/api/storageservices/put-page-from-url.
func (pb PageBlobURL) UploadPagesFromURL(ctx context.Context, sourceURL url.URL, sourceOffset int64, destOffset int64, count int64, ac PageBlobAccessConditions, transactionalMD5 []byte) (*PageBlobUploadPagesFromURLResponse, error) {
ifModifiedSince, ifUnmodifiedSince, ifMatchETag, ifNoneMatchETag := ac.ModifiedAccessConditions.pointers()
ifSequenceNumberLessThanOrEqual, ifSequenceNumberLessThan, ifSequenceNumberEqual := ac.SequenceNumberAccessConditions.pointers()
func (pb PageBlobURL) UploadPagesFromURL(ctx context.Context, sourceURL url.URL, sourceOffset int64, destOffset int64, count int64, transactionalMD5 []byte, destinationAccessConditions PageBlobAccessConditions, sourceAccessConditions ModifiedAccessConditions) (*PageBlobUploadPagesFromURLResponse, error) {
ifModifiedSince, ifUnmodifiedSince, ifMatchETag, ifNoneMatchETag := destinationAccessConditions.ModifiedAccessConditions.pointers()
sourceIfModifiedSince, sourceIfUnmodifiedSince, sourceIfMatchETag, sourceIfNoneMatchETag := sourceAccessConditions.pointers()
ifSequenceNumberLessThanOrEqual, ifSequenceNumberLessThan, ifSequenceNumberEqual := destinationAccessConditions.SequenceNumberAccessConditions.pointers()
return pb.pbClient.UploadPagesFromURL(ctx, sourceURL.String(), *PageRange{Start: sourceOffset, End: sourceOffset + count - 1}.pointers(), 0,
*PageRange{Start: destOffset, End: destOffset + count - 1}.pointers(), transactionalMD5, nil, ac.LeaseAccessConditions.pointers(),
*PageRange{Start: destOffset, End: destOffset + count - 1}.pointers(), transactionalMD5, nil, destinationAccessConditions.LeaseAccessConditions.pointers(),
ifSequenceNumberLessThanOrEqual, ifSequenceNumberLessThan, ifSequenceNumberEqual,
ifModifiedSince, ifUnmodifiedSince, ifMatchETag, ifNoneMatchETag, nil)
ifModifiedSince, ifUnmodifiedSince, ifMatchETag, ifNoneMatchETag, sourceIfModifiedSince, sourceIfUnmodifiedSince, sourceIfMatchETag, sourceIfNoneMatchETag, nil)
}
// ClearPages frees the specified pages from the page blob.

View file

@ -27,6 +27,17 @@ func NewServiceURL(primaryURL url.URL, p pipeline.Pipeline) ServiceURL {
return ServiceURL{client: client}
}
//GetUserDelegationCredential obtains a UserDelegationKey object using the base ServiceURL object.
//OAuth is required for this call, as well as any role that can delegate access to the storage account.
func (s ServiceURL) GetUserDelegationCredential(ctx context.Context, info KeyInfo, timeout *int32, requestID *string) (UserDelegationCredential, error) {
sc := newServiceClient(s.client.url, s.client.p)
udk, err := sc.GetUserDelegationKey(ctx, info, timeout, requestID)
if err != nil {
return UserDelegationCredential{}, err
}
return NewUserDelegationCredential(strings.Split(s.client.url.Host, ".")[0], *udk), nil
}
// URL returns the URL endpoint used by the ServiceURL object.
func (s ServiceURL) URL() url.URL {
return s.client.URL()

View file

@ -0,0 +1,38 @@
package azblob
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
)
// NewUserDelegationCredential creates a new UserDelegationCredential using a Storage account's name and a user delegation key from it
func NewUserDelegationCredential(accountName string, key UserDelegationKey) UserDelegationCredential {
return UserDelegationCredential{
accountName: accountName,
accountKey: key,
}
}
type UserDelegationCredential struct {
accountName string
accountKey UserDelegationKey
}
// AccountName returns the Storage account's name
func (f UserDelegationCredential) AccountName() string {
return f.accountName
}
// ComputeHMAC
func (f UserDelegationCredential) ComputeHMACSHA256(message string) (base64String string) {
bytes, _ := base64.StdEncoding.DecodeString(f.accountKey.Value)
h := hmac.New(sha256.New, bytes)
h.Write([]byte(message))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
// Private method to return important parameters for NewSASQueryParameters
func (f UserDelegationCredential) getUDKParams() *UserDelegationKey {
return &f.accountKey
}

View file

@ -1,3 +1,3 @@
package azblob
const serviceLibVersion = "0.6"
const serviceLibVersion = "0.7"

View file

@ -39,6 +39,15 @@ func (f SharedKeyCredential) AccountName() string {
return f.accountName
}
func (f SharedKeyCredential) getAccountKey() []byte {
return f.accountKey
}
// noop function to satisfy StorageAccountCredential interface
func (f SharedKeyCredential) getUDKParams() *UserDelegationKey {
return nil
}
// New creates a credential policy object.
func (f *SharedKeyCredential) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy {
return pipeline.PolicyFunc(func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
@ -88,7 +97,7 @@ const (
)
// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.
func (f *SharedKeyCredential) ComputeHMACSHA256(message string) (base64String string) {
func (f SharedKeyCredential) ComputeHMACSHA256(message string) (base64String string) {
h := hmac.New(sha256.New, f.accountKey)
h.Write([]byte(message))
return base64.StdEncoding.EncodeToString(h.Sum(nil))

View file

@ -176,7 +176,8 @@ func NewRetryPolicyFactory(o RetryOptions) pipeline.Factory {
}
if !tryingPrimary {
requestCopy.Request.URL.Host = o.retryReadsFromSecondaryHost()
requestCopy.URL.Host = o.retryReadsFromSecondaryHost()
requestCopy.Host = o.retryReadsFromSecondaryHost()
}
// Set the server-side timeout query parameter "timeout=[seconds]"
@ -211,7 +212,7 @@ func NewRetryPolicyFactory(o RetryOptions) pipeline.Factory {
switch {
case ctx.Err() != nil:
action = "NoRetry: Op timeout"
case !tryingPrimary && response != nil && response.Response().StatusCode == http.StatusNotFound:
case !tryingPrimary && response != nil && response.Response() != nil && response.Response().StatusCode == http.StatusNotFound:
// If attempt was against the secondary & it returned a StatusNotFound (404), then
// the resource was not found. This may be due to replication delay. So, in this
// case, we'll never try the secondary again for this operation.

View file

@ -37,7 +37,7 @@ func (v AccountSASSignatureValues) NewSASQueryParameters(sharedKeyCredential *Sh
}
v.Permissions = perms.String()
startTime, expiryTime := FormatTimesForSASSigning(v.StartTime, v.ExpiryTime)
startTime, expiryTime, _ := FormatTimesForSASSigning(v.StartTime, v.ExpiryTime, time.Time{})
stringToSign := strings.Join([]string{
sharedKeyCredential.AccountName(),
@ -69,6 +69,7 @@ func (v AccountSASSignatureValues) NewSASQueryParameters(sharedKeyCredential *Sh
// Calculated SAS signature
signature: signature,
}
return p, nil
}

View file

@ -22,7 +22,7 @@ const (
// FormatTimesForSASSigning converts a time.Time to a snapshotTimeFormat string suitable for a
// SASField's StartTime or ExpiryTime fields. Returns "" if value.IsZero().
func FormatTimesForSASSigning(startTime, expiryTime time.Time) (string, string) {
func FormatTimesForSASSigning(startTime, expiryTime, snapshotTime time.Time) (string, string, string) {
ss := ""
if !startTime.IsZero() {
ss = startTime.Format(SASTimeFormat) // "yyyy-MM-ddTHH:mm:ssZ"
@ -31,7 +31,11 @@ func FormatTimesForSASSigning(startTime, expiryTime time.Time) (string, string)
if !expiryTime.IsZero() {
se = expiryTime.Format(SASTimeFormat) // "yyyy-MM-ddTHH:mm:ssZ"
}
return ss, se
sh := ""
if !snapshotTime.IsZero() {
sh = snapshotTime.Format(SnapshotTimeFormat)
}
return ss, se, sh
}
// SASTimeFormat represents the format of a SAS start or expiry time. Use it when formatting/parsing a time.Time.
@ -53,6 +57,7 @@ type SASQueryParameters struct {
protocol SASProtocol `param:"spr"`
startTime time.Time `param:"st"`
expiryTime time.Time `param:"se"`
snapshotTime time.Time `param:"snapshot"`
ipRange IPRange `param:"sip"`
identifier string `param:"si"`
resource string `param:"sr"`
@ -63,6 +68,40 @@ type SASQueryParameters struct {
contentEncoding string `param:"rsce"`
contentLanguage string `param:"rscl"`
contentType string `param:"rsct"`
signedOid string `param:"skoid"`
signedTid string `param:"sktid"`
signedStart time.Time `param:"skt"`
signedExpiry time.Time `param:"ske"`
signedService string `param:"sks"`
signedVersion string `param:"skv"`
}
func (p *SASQueryParameters) SignedOid() string {
return p.signedOid
}
func (p *SASQueryParameters) SignedTid() string {
return p.signedTid
}
func (p *SASQueryParameters) SignedStart() time.Time {
return p.signedStart
}
func (p *SASQueryParameters) SignedExpiry() time.Time {
return p.signedExpiry
}
func (p *SASQueryParameters) SignedService() string {
return p.signedService
}
func (p *SASQueryParameters) SignedVersion() string {
return p.signedVersion
}
func (p *SASQueryParameters) SnapshotTime() time.Time {
return p.snapshotTime
}
func (p *SASQueryParameters) Version() string {
@ -160,6 +199,8 @@ func newSASQueryParameters(values url.Values, deleteSASParametersFromValues bool
p.resourceTypes = val
case "spr":
p.protocol = SASProtocol(val)
case "snapshot":
p.snapshotTime, _ = time.Parse(SnapshotTimeFormat, val)
case "st":
p.startTime, _ = time.Parse(SASTimeFormat, val)
case "se":
@ -190,6 +231,18 @@ func newSASQueryParameters(values url.Values, deleteSASParametersFromValues bool
p.contentLanguage = val
case "rsct":
p.contentType = val
case "skoid":
p.signedOid = val
case "sktid":
p.signedTid = val
case "skt":
p.signedStart, _ = time.Parse(SASTimeFormat, val)
case "ske":
p.signedExpiry, _ = time.Parse(SASTimeFormat, val)
case "sks":
p.signedService = val
case "skv":
p.signedVersion = val
default:
isSASKey = false // We didn't recognize the query parameter
}
@ -232,6 +285,14 @@ func (p *SASQueryParameters) addToValues(v url.Values) url.Values {
if p.permissions != "" {
v.Add("sp", p.permissions)
}
if p.signedOid != "" {
v.Add("skoid", p.signedOid)
v.Add("sktid", p.signedTid)
v.Add("skt", p.signedStart.Format(SASTimeFormat))
v.Add("ske", p.signedExpiry.Format(SASTimeFormat))
v.Add("sks", p.signedService)
v.Add("skv", p.signedVersion)
}
if p.signature != "" {
v.Add("sig", p.signature)
}

View file

@ -130,28 +130,31 @@ func (client appendBlobClient) appendBlockResponder(resp pipeline.Response) (pip
// source data in the specified range. sourceContentMD5 is specify the md5 calculated for the range of bytes that must
// be read from the copy source. timeout is the timeout parameter is expressed in seconds. For more information, see <a
// href="https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/setting-timeouts-for-blob-service-operations">Setting
// Timeouts for Blob Service Operations.</a> transactionalContentMD5 is specify the transactional md5 for the body, to
// be validated by the service. leaseID is if specified, the operation only succeeds if the resource's lease is active
// and matches this ID. maxSize is optional conditional header. The max length in bytes permitted for the append blob.
// If the Append Block operation would cause the blob to exceed that limit or if the blob size is already greater than
// the value specified in this header, the request will fail with MaxBlobSizeConditionNotMet error (HTTP status code
// 412 - Precondition Failed). appendPosition is optional conditional header, used only for the Append Block operation.
// A number indicating the byte offset to compare. Append Block will succeed only if the append position is equal to
// this number. If it is not, the request will fail with the AppendPositionConditionNotMet error (HTTP status code 412
// - Precondition Failed). ifModifiedSince is specify this header value to operate only on a blob if it has been
// modified since the specified date/time. ifUnmodifiedSince is specify this header value to operate only on a blob if
// it has not been modified since the specified date/time. ifMatch is specify an ETag value to operate only on blobs
// with a matching value. ifNoneMatch is specify an ETag value to operate only on blobs without a matching value.
// Timeouts for Blob Service Operations.</a> leaseID is if specified, the operation only succeeds if the resource's
// lease is active and matches this ID. maxSize is optional conditional header. The max length in bytes permitted for
// the append blob. If the Append Block operation would cause the blob to exceed that limit or if the blob size is
// already greater than the value specified in this header, the request will fail with MaxBlobSizeConditionNotMet error
// (HTTP status code 412 - Precondition Failed). appendPosition is optional conditional header, used only for the
// Append Block operation. A number indicating the byte offset to compare. Append Block will succeed only if the append
// position is equal to this number. If it is not, the request will fail with the AppendPositionConditionNotMet error
// (HTTP status code 412 - Precondition Failed). ifModifiedSince is specify this header value to operate only on a blob
// if it has been modified since the specified date/time. ifUnmodifiedSince is specify this header value to operate
// only on a blob if it has not been modified since the specified date/time. ifMatch is specify an ETag value to
// operate only on blobs with a matching value. ifNoneMatch is specify an ETag value to operate only on blobs without a
// matching value. sourceIfModifiedSince is specify this header value to operate only on a blob if it has been modified
// since the specified date/time. sourceIfUnmodifiedSince is specify this header value to operate only on a blob if it
// has not been modified since the specified date/time. sourceIfMatch is specify an ETag value to operate only on blobs
// with a matching value. sourceIfNoneMatch is specify an ETag value to operate only on blobs without a matching value.
// requestID is provides a client-generated, opaque value with a 1 KB character limit that is recorded in the analytics
// logs when storage analytics logging is enabled.
func (client appendBlobClient) AppendBlockFromURL(ctx context.Context, sourceURL string, contentLength int64, sourceRange *string, sourceContentMD5 []byte, timeout *int32, transactionalContentMD5 []byte, leaseID *string, maxSize *int64, appendPosition *int64, ifModifiedSince *time.Time, ifUnmodifiedSince *time.Time, ifMatch *ETag, ifNoneMatch *ETag, requestID *string) (*AppendBlobAppendBlockFromURLResponse, error) {
func (client appendBlobClient) AppendBlockFromURL(ctx context.Context, sourceURL string, contentLength int64, sourceRange *string, sourceContentMD5 []byte, timeout *int32, leaseID *string, maxSize *int64, appendPosition *int64, ifModifiedSince *time.Time, ifUnmodifiedSince *time.Time, ifMatch *ETag, ifNoneMatch *ETag, sourceIfModifiedSince *time.Time, sourceIfUnmodifiedSince *time.Time, sourceIfMatch *ETag, sourceIfNoneMatch *ETag, requestID *string) (*AppendBlobAppendBlockFromURLResponse, error) {
if err := validate([]validation{
{targetValue: timeout,
constraints: []constraint{{target: "timeout", name: null, rule: false,
chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 0, chain: nil}}}}}}); err != nil {
return nil, err
}
req, err := client.appendBlockFromURLPreparer(sourceURL, contentLength, sourceRange, sourceContentMD5, timeout, transactionalContentMD5, leaseID, maxSize, appendPosition, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, requestID)
req, err := client.appendBlockFromURLPreparer(sourceURL, contentLength, sourceRange, sourceContentMD5, timeout, leaseID, maxSize, appendPosition, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, sourceIfModifiedSince, sourceIfUnmodifiedSince, sourceIfMatch, sourceIfNoneMatch, requestID)
if err != nil {
return nil, err
}
@ -163,7 +166,7 @@ func (client appendBlobClient) AppendBlockFromURL(ctx context.Context, sourceURL
}
// appendBlockFromURLPreparer prepares the AppendBlockFromURL request.
func (client appendBlobClient) appendBlockFromURLPreparer(sourceURL string, contentLength int64, sourceRange *string, sourceContentMD5 []byte, timeout *int32, transactionalContentMD5 []byte, leaseID *string, maxSize *int64, appendPosition *int64, ifModifiedSince *time.Time, ifUnmodifiedSince *time.Time, ifMatch *ETag, ifNoneMatch *ETag, requestID *string) (pipeline.Request, error) {
func (client appendBlobClient) appendBlockFromURLPreparer(sourceURL string, contentLength int64, sourceRange *string, sourceContentMD5 []byte, timeout *int32, leaseID *string, maxSize *int64, appendPosition *int64, ifModifiedSince *time.Time, ifUnmodifiedSince *time.Time, ifMatch *ETag, ifNoneMatch *ETag, sourceIfModifiedSince *time.Time, sourceIfUnmodifiedSince *time.Time, sourceIfMatch *ETag, sourceIfNoneMatch *ETag, requestID *string) (pipeline.Request, error) {
req, err := pipeline.NewRequest("PUT", client.url, nil)
if err != nil {
return req, pipeline.NewError(err, "failed to create request")
@ -182,9 +185,6 @@ func (client appendBlobClient) appendBlockFromURLPreparer(sourceURL string, cont
req.Header.Set("x-ms-source-content-md5", base64.StdEncoding.EncodeToString(sourceContentMD5))
}
req.Header.Set("Content-Length", strconv.FormatInt(contentLength, 10))
if transactionalContentMD5 != nil {
req.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(transactionalContentMD5))
}
if leaseID != nil {
req.Header.Set("x-ms-lease-id", *leaseID)
}
@ -206,6 +206,18 @@ func (client appendBlobClient) appendBlockFromURLPreparer(sourceURL string, cont
if ifNoneMatch != nil {
req.Header.Set("If-None-Match", string(*ifNoneMatch))
}
if sourceIfModifiedSince != nil {
req.Header.Set("x-ms-source-if-modified-since", (*sourceIfModifiedSince).In(gmt).Format(time.RFC1123))
}
if sourceIfUnmodifiedSince != nil {
req.Header.Set("x-ms-source-if-unmodified-since", (*sourceIfUnmodifiedSince).In(gmt).Format(time.RFC1123))
}
if sourceIfMatch != nil {
req.Header.Set("x-ms-source-if-match", string(*sourceIfMatch))
}
if sourceIfNoneMatch != nil {
req.Header.Set("x-ms-source-if-none-match", string(*sourceIfNoneMatch))
}
req.Header.Set("x-ms-version", ServiceVersion)
if requestID != nil {
req.Header.Set("x-ms-client-request-id", *requestID)

View file

@ -6,14 +6,13 @@ package azblob
import (
"context"
"encoding/base64"
"github.com/Azure/azure-pipeline-go/pipeline"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"time"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// blobClient is the client for the Blob methods of the Azblob service.
@ -327,6 +326,111 @@ func (client blobClient) changeLeaseResponder(resp pipeline.Response) (pipeline.
return &BlobChangeLeaseResponse{rawResponse: resp.Response()}, err
}
// CopyFromURL the Copy From URL operation copies a blob or an internet resource to a new blob. It will not return a
// response until the copy is complete.
//
// copySource is specifies the name of the source page blob snapshot. This value is a URL of up to 2 KB in length that
// specifies a page blob snapshot. The value should be URL-encoded as it would appear in a request URI. The source blob
// must either be public or must be authenticated via a shared access signature. timeout is the timeout parameter is
// expressed in seconds. For more information, see <a
// href="https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/setting-timeouts-for-blob-service-operations">Setting
// Timeouts for Blob Service Operations.</a> metadata is optional. Specifies a user-defined name-value pair associated
// with the blob. If no name-value pairs are specified, the operation will copy the metadata from the source blob or
// file to the destination blob. If one or more name-value pairs are specified, the destination blob is created with
// the specified metadata, and metadata is not copied from the source blob or file. Note that beginning with version
// 2009-09-19, metadata names must adhere to the naming rules for C# identifiers. See Naming and Referencing
// Containers, Blobs, and Metadata for more information. sourceIfModifiedSince is specify this header value to operate
// only on a blob if it has been modified since the specified date/time. sourceIfUnmodifiedSince is specify this header
// value to operate only on a blob if it has not been modified since the specified date/time. sourceIfMatch is specify
// an ETag value to operate only on blobs with a matching value. sourceIfNoneMatch is specify an ETag value to operate
// only on blobs without a matching value. ifModifiedSince is specify this header value to operate only on a blob if it
// has been modified since the specified date/time. ifUnmodifiedSince is specify this header value to operate only on a
// blob if it has not been modified since the specified date/time. ifMatch is specify an ETag value to operate only on
// blobs with a matching value. ifNoneMatch is specify an ETag value to operate only on blobs without a matching value.
// leaseID is if specified, the operation only succeeds if the resource's lease is active and matches this ID.
// requestID is provides a client-generated, opaque value with a 1 KB character limit that is recorded in the analytics
// logs when storage analytics logging is enabled.
func (client blobClient) CopyFromURL(ctx context.Context, copySource string, timeout *int32, metadata map[string]string, sourceIfModifiedSince *time.Time, sourceIfUnmodifiedSince *time.Time, sourceIfMatch *ETag, sourceIfNoneMatch *ETag, ifModifiedSince *time.Time, ifUnmodifiedSince *time.Time, ifMatch *ETag, ifNoneMatch *ETag, leaseID *string, requestID *string) (*BlobCopyFromURLResponse, error) {
if err := validate([]validation{
{targetValue: timeout,
constraints: []constraint{{target: "timeout", name: null, rule: false,
chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 0, chain: nil}}}}}}); err != nil {
return nil, err
}
req, err := client.copyFromURLPreparer(copySource, timeout, metadata, sourceIfModifiedSince, sourceIfUnmodifiedSince, sourceIfMatch, sourceIfNoneMatch, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, leaseID, requestID)
if err != nil {
return nil, err
}
resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.copyFromURLResponder}, req)
if err != nil {
return nil, err
}
return resp.(*BlobCopyFromURLResponse), err
}
// copyFromURLPreparer prepares the CopyFromURL request.
func (client blobClient) copyFromURLPreparer(copySource string, timeout *int32, metadata map[string]string, sourceIfModifiedSince *time.Time, sourceIfUnmodifiedSince *time.Time, sourceIfMatch *ETag, sourceIfNoneMatch *ETag, ifModifiedSince *time.Time, ifUnmodifiedSince *time.Time, ifMatch *ETag, ifNoneMatch *ETag, leaseID *string, requestID *string) (pipeline.Request, error) {
req, err := pipeline.NewRequest("PUT", client.url, nil)
if err != nil {
return req, pipeline.NewError(err, "failed to create request")
}
params := req.URL.Query()
if timeout != nil {
params.Set("timeout", strconv.FormatInt(int64(*timeout), 10))
}
req.URL.RawQuery = params.Encode()
if metadata != nil {
for k, v := range metadata {
req.Header.Set("x-ms-meta-"+k, v)
}
}
if sourceIfModifiedSince != nil {
req.Header.Set("x-ms-source-if-modified-since", (*sourceIfModifiedSince).In(gmt).Format(time.RFC1123))
}
if sourceIfUnmodifiedSince != nil {
req.Header.Set("x-ms-source-if-unmodified-since", (*sourceIfUnmodifiedSince).In(gmt).Format(time.RFC1123))
}
if sourceIfMatch != nil {
req.Header.Set("x-ms-source-if-match", string(*sourceIfMatch))
}
if sourceIfNoneMatch != nil {
req.Header.Set("x-ms-source-if-none-match", string(*sourceIfNoneMatch))
}
if ifModifiedSince != nil {
req.Header.Set("If-Modified-Since", (*ifModifiedSince).In(gmt).Format(time.RFC1123))
}
if ifUnmodifiedSince != nil {
req.Header.Set("If-Unmodified-Since", (*ifUnmodifiedSince).In(gmt).Format(time.RFC1123))
}
if ifMatch != nil {
req.Header.Set("If-Match", string(*ifMatch))
}
if ifNoneMatch != nil {
req.Header.Set("If-None-Match", string(*ifNoneMatch))
}
req.Header.Set("x-ms-copy-source", copySource)
if leaseID != nil {
req.Header.Set("x-ms-lease-id", *leaseID)
}
req.Header.Set("x-ms-version", ServiceVersion)
if requestID != nil {
req.Header.Set("x-ms-client-request-id", *requestID)
}
req.Header.Set("x-ms-requires-sync", "true")
return req, nil
}
// copyFromURLResponder handles the response to the CopyFromURL request.
func (client blobClient) copyFromURLResponder(resp pipeline.Response) (pipeline.Response, error) {
err := validateResponse(resp, http.StatusOK, http.StatusAccepted)
if resp == nil {
return nil, err
}
io.Copy(ioutil.Discard, resp.Response().Body)
resp.Response().Body.Close()
return &BlobCopyFromURLResponse{rawResponse: resp.Response()}, err
}
// CreateSnapshot the Create Snapshot operation creates a read-only snapshot of a blob
//
// timeout is the timeout parameter is expressed in seconds. For more information, see <a

View file

@ -8,14 +8,13 @@ import (
"context"
"encoding/base64"
"encoding/xml"
"github.com/Azure/azure-pipeline-go/pipeline"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"time"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// blockBlobClient is the client for the BlockBlob methods of the Azblob service.
@ -314,16 +313,20 @@ func (client blockBlobClient) stageBlockResponder(resp pipeline.Response) (pipel
// more information, see <a
// href="https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/setting-timeouts-for-blob-service-operations">Setting
// Timeouts for Blob Service Operations.</a> leaseID is if specified, the operation only succeeds if the resource's
// lease is active and matches this ID. requestID is provides a client-generated, opaque value with a 1 KB character
// limit that is recorded in the analytics logs when storage analytics logging is enabled.
func (client blockBlobClient) StageBlockFromURL(ctx context.Context, blockID string, contentLength int64, sourceURL string, sourceRange *string, sourceContentMD5 []byte, timeout *int32, leaseID *string, requestID *string) (*BlockBlobStageBlockFromURLResponse, error) {
// lease is active and matches this ID. sourceIfModifiedSince is specify this header value to operate only on a blob if
// it has been modified since the specified date/time. sourceIfUnmodifiedSince is specify this header value to operate
// only on a blob if it has not been modified since the specified date/time. sourceIfMatch is specify an ETag value to
// operate only on blobs with a matching value. sourceIfNoneMatch is specify an ETag value to operate only on blobs
// without a matching value. requestID is provides a client-generated, opaque value with a 1 KB character limit that is
// recorded in the analytics logs when storage analytics logging is enabled.
func (client blockBlobClient) StageBlockFromURL(ctx context.Context, blockID string, contentLength int64, sourceURL string, sourceRange *string, sourceContentMD5 []byte, timeout *int32, leaseID *string, sourceIfModifiedSince *time.Time, sourceIfUnmodifiedSince *time.Time, sourceIfMatch *ETag, sourceIfNoneMatch *ETag, requestID *string) (*BlockBlobStageBlockFromURLResponse, error) {
if err := validate([]validation{
{targetValue: timeout,
constraints: []constraint{{target: "timeout", name: null, rule: false,
chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 0, chain: nil}}}}}}); err != nil {
return nil, err
}
req, err := client.stageBlockFromURLPreparer(blockID, contentLength, sourceURL, sourceRange, sourceContentMD5, timeout, leaseID, requestID)
req, err := client.stageBlockFromURLPreparer(blockID, contentLength, sourceURL, sourceRange, sourceContentMD5, timeout, leaseID, sourceIfModifiedSince, sourceIfUnmodifiedSince, sourceIfMatch, sourceIfNoneMatch, requestID)
if err != nil {
return nil, err
}
@ -335,7 +338,7 @@ func (client blockBlobClient) StageBlockFromURL(ctx context.Context, blockID str
}
// stageBlockFromURLPreparer prepares the StageBlockFromURL request.
func (client blockBlobClient) stageBlockFromURLPreparer(blockID string, contentLength int64, sourceURL string, sourceRange *string, sourceContentMD5 []byte, timeout *int32, leaseID *string, requestID *string) (pipeline.Request, error) {
func (client blockBlobClient) stageBlockFromURLPreparer(blockID string, contentLength int64, sourceURL string, sourceRange *string, sourceContentMD5 []byte, timeout *int32, leaseID *string, sourceIfModifiedSince *time.Time, sourceIfUnmodifiedSince *time.Time, sourceIfMatch *ETag, sourceIfNoneMatch *ETag, requestID *string) (pipeline.Request, error) {
req, err := pipeline.NewRequest("PUT", client.url, nil)
if err != nil {
return req, pipeline.NewError(err, "failed to create request")
@ -358,6 +361,18 @@ func (client blockBlobClient) stageBlockFromURLPreparer(blockID string, contentL
if leaseID != nil {
req.Header.Set("x-ms-lease-id", *leaseID)
}
if sourceIfModifiedSince != nil {
req.Header.Set("x-ms-source-if-modified-since", (*sourceIfModifiedSince).In(gmt).Format(time.RFC1123))
}
if sourceIfUnmodifiedSince != nil {
req.Header.Set("x-ms-source-if-unmodified-since", (*sourceIfUnmodifiedSince).In(gmt).Format(time.RFC1123))
}
if sourceIfMatch != nil {
req.Header.Set("x-ms-source-if-match", string(*sourceIfMatch))
}
if sourceIfNoneMatch != nil {
req.Header.Set("x-ms-source-if-none-match", string(*sourceIfNoneMatch))
}
req.Header.Set("x-ms-version", ServiceVersion)
if requestID != nil {
req.Header.Set("x-ms-client-request-id", *requestID)

View file

@ -4,9 +4,8 @@ package azblob
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
import (
"net/url"
"github.com/Azure/azure-pipeline-go/pipeline"
"net/url"
)
const (

View file

@ -7,14 +7,13 @@ import (
"bytes"
"context"
"encoding/xml"
"github.com/Azure/azure-pipeline-go/pipeline"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"time"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// containerClient is the client for the Container methods of the Azblob service.

View file

@ -4,6 +4,8 @@ package azblob
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/xml"
"errors"
@ -425,6 +427,8 @@ const (
StorageErrorCodeAppendPositionConditionNotMet StorageErrorCodeType = "AppendPositionConditionNotMet"
// StorageErrorCodeAuthenticationFailed ...
StorageErrorCodeAuthenticationFailed StorageErrorCodeType = "AuthenticationFailed"
// StorageErrorCodeAuthorizationFailure ...
StorageErrorCodeAuthorizationFailure StorageErrorCodeType = "AuthorizationFailure"
// StorageErrorCodeBlobAlreadyExists ...
StorageErrorCodeBlobAlreadyExists StorageErrorCodeType = "BlobAlreadyExists"
// StorageErrorCodeBlobArchived ...
@ -629,7 +633,22 @@ const (
// PossibleStorageErrorCodeTypeValues returns an array of possible values for the StorageErrorCodeType const type.
func PossibleStorageErrorCodeTypeValues() []StorageErrorCodeType {
return []StorageErrorCodeType{StorageErrorCodeAccountAlreadyExists, StorageErrorCodeAccountBeingCreated, StorageErrorCodeAccountIsDisabled, StorageErrorCodeAppendPositionConditionNotMet, StorageErrorCodeAuthenticationFailed, StorageErrorCodeBlobAlreadyExists, StorageErrorCodeBlobArchived, StorageErrorCodeBlobBeingRehydrated, StorageErrorCodeBlobNotArchived, StorageErrorCodeBlobNotFound, StorageErrorCodeBlobOverwritten, StorageErrorCodeBlobTierInadequateForContentLength, StorageErrorCodeBlockCountExceedsLimit, StorageErrorCodeBlockListTooLong, StorageErrorCodeCannotChangeToLowerTier, StorageErrorCodeCannotVerifyCopySource, StorageErrorCodeConditionHeadersNotSupported, StorageErrorCodeConditionNotMet, StorageErrorCodeContainerAlreadyExists, StorageErrorCodeContainerBeingDeleted, StorageErrorCodeContainerDisabled, StorageErrorCodeContainerNotFound, StorageErrorCodeContentLengthLargerThanTierLimit, StorageErrorCodeCopyAcrossAccountsNotSupported, StorageErrorCodeCopyIDMismatch, StorageErrorCodeEmptyMetadataKey, StorageErrorCodeFeatureVersionMismatch, StorageErrorCodeIncrementalCopyBlobMismatch, StorageErrorCodeIncrementalCopyOfEralierVersionSnapshotNotAllowed, StorageErrorCodeIncrementalCopySourceMustBeSnapshot, StorageErrorCodeInfiniteLeaseDurationRequired, StorageErrorCodeInsufficientAccountPermissions, StorageErrorCodeInternalError, StorageErrorCodeInvalidAuthenticationInfo, StorageErrorCodeInvalidBlobOrBlock, StorageErrorCodeInvalidBlobTier, StorageErrorCodeInvalidBlobType, StorageErrorCodeInvalidBlockID, StorageErrorCodeInvalidBlockList, StorageErrorCodeInvalidHeaderValue, StorageErrorCodeInvalidHTTPVerb, StorageErrorCodeInvalidInput, StorageErrorCodeInvalidMd5, StorageErrorCodeInvalidMetadata, StorageErrorCodeInvalidOperation, StorageErrorCodeInvalidPageRange, StorageErrorCodeInvalidQueryParameterValue, StorageErrorCodeInvalidRange, StorageErrorCodeInvalidResourceName, StorageErrorCodeInvalidSourceBlobType, StorageErrorCodeInvalidSourceBlobURL, StorageErrorCodeInvalidURI, StorageErrorCodeInvalidVersionForPageBlobOperation, StorageErrorCodeInvalidXMLDocument, StorageErrorCodeInvalidXMLNodeValue, StorageErrorCodeLeaseAlreadyBroken, StorageErrorCodeLeaseAlreadyPresent, StorageErrorCodeLeaseIDMismatchWithBlobOperation, StorageErrorCodeLeaseIDMismatchWithContainerOperation, StorageErrorCodeLeaseIDMismatchWithLeaseOperation, StorageErrorCodeLeaseIDMissing, StorageErrorCodeLeaseIsBreakingAndCannotBeAcquired, StorageErrorCodeLeaseIsBreakingAndCannotBeChanged, StorageErrorCodeLeaseIsBrokenAndCannotBeRenewed, StorageErrorCodeLeaseLost, StorageErrorCodeLeaseNotPresentWithBlobOperation, StorageErrorCodeLeaseNotPresentWithContainerOperation, StorageErrorCodeLeaseNotPresentWithLeaseOperation, StorageErrorCodeMaxBlobSizeConditionNotMet, StorageErrorCodeMd5Mismatch, StorageErrorCodeMetadataTooLarge, StorageErrorCodeMissingContentLengthHeader, StorageErrorCodeMissingRequiredHeader, StorageErrorCodeMissingRequiredQueryParameter, StorageErrorCodeMissingRequiredXMLNode, StorageErrorCodeMultipleConditionHeadersNotSupported, StorageErrorCodeNone, StorageErrorCodeNoPendingCopyOperation, StorageErrorCodeOperationNotAllowedOnIncrementalCopyBlob, StorageErrorCodeOperationTimedOut, StorageErrorCodeOutOfRangeInput, StorageErrorCodeOutOfRangeQueryParameterValue, StorageErrorCodePendingCopyOperation, StorageErrorCodePreviousSnapshotCannotBeNewer, StorageErrorCodePreviousSnapshotNotFound, StorageErrorCodePreviousSnapshotOperationNotSupported, StorageErrorCodeRequestBodyTooLarge, StorageErrorCodeRequestURLFailedToParse, StorageErrorCodeResourceAlreadyExists, StorageErrorCodeResourceNotFound, StorageErrorCodeResourceTypeMismatch, StorageErrorCodeSequenceNumberConditionNotMet, StorageErrorCodeSequenceNumberIncrementTooLarge, StorageErrorCodeServerBusy, StorageErrorCodeSnaphotOperationRateExceeded, StorageErrorCodeSnapshotCountExceeded, StorageErrorCodeSnapshotsPresent, StorageErrorCodeSourceConditionNotMet, StorageErrorCodeSystemInUse, StorageErrorCodeTargetConditionNotMet, StorageErrorCodeUnauthorizedBlobOverwrite, StorageErrorCodeUnsupportedHeader, StorageErrorCodeUnsupportedHTTPVerb, StorageErrorCodeUnsupportedQueryParameter, StorageErrorCodeUnsupportedXMLNode}
return []StorageErrorCodeType{StorageErrorCodeAccountAlreadyExists, StorageErrorCodeAccountBeingCreated, StorageErrorCodeAccountIsDisabled, StorageErrorCodeAppendPositionConditionNotMet, StorageErrorCodeAuthenticationFailed, StorageErrorCodeAuthorizationFailure, StorageErrorCodeBlobAlreadyExists, StorageErrorCodeBlobArchived, StorageErrorCodeBlobBeingRehydrated, StorageErrorCodeBlobNotArchived, StorageErrorCodeBlobNotFound, StorageErrorCodeBlobOverwritten, StorageErrorCodeBlobTierInadequateForContentLength, StorageErrorCodeBlockCountExceedsLimit, StorageErrorCodeBlockListTooLong, StorageErrorCodeCannotChangeToLowerTier, StorageErrorCodeCannotVerifyCopySource, StorageErrorCodeConditionHeadersNotSupported, StorageErrorCodeConditionNotMet, StorageErrorCodeContainerAlreadyExists, StorageErrorCodeContainerBeingDeleted, StorageErrorCodeContainerDisabled, StorageErrorCodeContainerNotFound, StorageErrorCodeContentLengthLargerThanTierLimit, StorageErrorCodeCopyAcrossAccountsNotSupported, StorageErrorCodeCopyIDMismatch, StorageErrorCodeEmptyMetadataKey, StorageErrorCodeFeatureVersionMismatch, StorageErrorCodeIncrementalCopyBlobMismatch, StorageErrorCodeIncrementalCopyOfEralierVersionSnapshotNotAllowed, StorageErrorCodeIncrementalCopySourceMustBeSnapshot, StorageErrorCodeInfiniteLeaseDurationRequired, StorageErrorCodeInsufficientAccountPermissions, StorageErrorCodeInternalError, StorageErrorCodeInvalidAuthenticationInfo, StorageErrorCodeInvalidBlobOrBlock, StorageErrorCodeInvalidBlobTier, StorageErrorCodeInvalidBlobType, StorageErrorCodeInvalidBlockID, StorageErrorCodeInvalidBlockList, StorageErrorCodeInvalidHeaderValue, StorageErrorCodeInvalidHTTPVerb, StorageErrorCodeInvalidInput, StorageErrorCodeInvalidMd5, StorageErrorCodeInvalidMetadata, StorageErrorCodeInvalidOperation, StorageErrorCodeInvalidPageRange, StorageErrorCodeInvalidQueryParameterValue, StorageErrorCodeInvalidRange, StorageErrorCodeInvalidResourceName, StorageErrorCodeInvalidSourceBlobType, StorageErrorCodeInvalidSourceBlobURL, StorageErrorCodeInvalidURI, StorageErrorCodeInvalidVersionForPageBlobOperation, StorageErrorCodeInvalidXMLDocument, StorageErrorCodeInvalidXMLNodeValue, StorageErrorCodeLeaseAlreadyBroken, StorageErrorCodeLeaseAlreadyPresent, StorageErrorCodeLeaseIDMismatchWithBlobOperation, StorageErrorCodeLeaseIDMismatchWithContainerOperation, StorageErrorCodeLeaseIDMismatchWithLeaseOperation, StorageErrorCodeLeaseIDMissing, StorageErrorCodeLeaseIsBreakingAndCannotBeAcquired, StorageErrorCodeLeaseIsBreakingAndCannotBeChanged, StorageErrorCodeLeaseIsBrokenAndCannotBeRenewed, StorageErrorCodeLeaseLost, StorageErrorCodeLeaseNotPresentWithBlobOperation, StorageErrorCodeLeaseNotPresentWithContainerOperation, StorageErrorCodeLeaseNotPresentWithLeaseOperation, StorageErrorCodeMaxBlobSizeConditionNotMet, StorageErrorCodeMd5Mismatch, StorageErrorCodeMetadataTooLarge, StorageErrorCodeMissingContentLengthHeader, StorageErrorCodeMissingRequiredHeader, StorageErrorCodeMissingRequiredQueryParameter, StorageErrorCodeMissingRequiredXMLNode, StorageErrorCodeMultipleConditionHeadersNotSupported, StorageErrorCodeNone, StorageErrorCodeNoPendingCopyOperation, StorageErrorCodeOperationNotAllowedOnIncrementalCopyBlob, StorageErrorCodeOperationTimedOut, StorageErrorCodeOutOfRangeInput, StorageErrorCodeOutOfRangeQueryParameterValue, StorageErrorCodePendingCopyOperation, StorageErrorCodePreviousSnapshotCannotBeNewer, StorageErrorCodePreviousSnapshotNotFound, StorageErrorCodePreviousSnapshotOperationNotSupported, StorageErrorCodeRequestBodyTooLarge, StorageErrorCodeRequestURLFailedToParse, StorageErrorCodeResourceAlreadyExists, StorageErrorCodeResourceNotFound, StorageErrorCodeResourceTypeMismatch, StorageErrorCodeSequenceNumberConditionNotMet, StorageErrorCodeSequenceNumberIncrementTooLarge, StorageErrorCodeServerBusy, StorageErrorCodeSnaphotOperationRateExceeded, StorageErrorCodeSnapshotCountExceeded, StorageErrorCodeSnapshotsPresent, StorageErrorCodeSourceConditionNotMet, StorageErrorCodeSystemInUse, StorageErrorCodeTargetConditionNotMet, StorageErrorCodeUnauthorizedBlobOverwrite, StorageErrorCodeUnsupportedHeader, StorageErrorCodeUnsupportedHTTPVerb, StorageErrorCodeUnsupportedQueryParameter, StorageErrorCodeUnsupportedXMLNode}
}
// SyncCopyStatusType enumerates the values for sync copy status type.
type SyncCopyStatusType string
const (
// SyncCopyStatusNone represents an empty SyncCopyStatusType.
SyncCopyStatusNone SyncCopyStatusType = ""
// SyncCopyStatusSuccess ...
SyncCopyStatusSuccess SyncCopyStatusType = "success"
)
// PossibleSyncCopyStatusTypeValues returns an array of possible values for the SyncCopyStatusType const type.
func PossibleSyncCopyStatusTypeValues() []SyncCopyStatusType {
return []SyncCopyStatusType{SyncCopyStatusNone, SyncCopyStatusSuccess}
}
// AccessPolicy - An Access policy
@ -825,6 +844,11 @@ func (ababr AppendBlobAppendBlockResponse) ETag() ETag {
return ETag(ababr.rawResponse.Header.Get("ETag"))
}
// IsServerEncrypted returns the value for header x-ms-request-server-encrypted.
func (ababr AppendBlobAppendBlockResponse) IsServerEncrypted() string {
return ababr.rawResponse.Header.Get("x-ms-request-server-encrypted")
}
// LastModified returns the value for header Last-Modified.
func (ababr AppendBlobAppendBlockResponse) LastModified() time.Time {
s := ababr.rawResponse.Header.Get("Last-Modified")
@ -1201,6 +1225,82 @@ func (bclr BlobChangeLeaseResponse) Version() string {
return bclr.rawResponse.Header.Get("x-ms-version")
}
// BlobCopyFromURLResponse ...
type BlobCopyFromURLResponse struct {
rawResponse *http.Response
}
// Response returns the raw HTTP response object.
func (bcfur BlobCopyFromURLResponse) Response() *http.Response {
return bcfur.rawResponse
}
// StatusCode returns the HTTP status code of the response, e.g. 200.
func (bcfur BlobCopyFromURLResponse) StatusCode() int {
return bcfur.rawResponse.StatusCode
}
// Status returns the HTTP status message of the response, e.g. "200 OK".
func (bcfur BlobCopyFromURLResponse) Status() string {
return bcfur.rawResponse.Status
}
// CopyID returns the value for header x-ms-copy-id.
func (bcfur BlobCopyFromURLResponse) CopyID() string {
return bcfur.rawResponse.Header.Get("x-ms-copy-id")
}
// CopyStatus returns the value for header x-ms-copy-status.
func (bcfur BlobCopyFromURLResponse) CopyStatus() SyncCopyStatusType {
return SyncCopyStatusType(bcfur.rawResponse.Header.Get("x-ms-copy-status"))
}
// Date returns the value for header Date.
func (bcfur BlobCopyFromURLResponse) Date() time.Time {
s := bcfur.rawResponse.Header.Get("Date")
if s == "" {
return time.Time{}
}
t, err := time.Parse(time.RFC1123, s)
if err != nil {
t = time.Time{}
}
return t
}
// ErrorCode returns the value for header x-ms-error-code.
func (bcfur BlobCopyFromURLResponse) ErrorCode() string {
return bcfur.rawResponse.Header.Get("x-ms-error-code")
}
// ETag returns the value for header ETag.
func (bcfur BlobCopyFromURLResponse) ETag() ETag {
return ETag(bcfur.rawResponse.Header.Get("ETag"))
}
// LastModified returns the value for header Last-Modified.
func (bcfur BlobCopyFromURLResponse) LastModified() time.Time {
s := bcfur.rawResponse.Header.Get("Last-Modified")
if s == "" {
return time.Time{}
}
t, err := time.Parse(time.RFC1123, s)
if err != nil {
t = time.Time{}
}
return t
}
// RequestID returns the value for header x-ms-request-id.
func (bcfur BlobCopyFromURLResponse) RequestID() string {
return bcfur.rawResponse.Header.Get("x-ms-request-id")
}
// Version returns the value for header x-ms-version.
func (bcfur BlobCopyFromURLResponse) Version() string {
return bcfur.rawResponse.Header.Get("x-ms-version")
}
// BlobCreateSnapshotResponse ...
type BlobCreateSnapshotResponse struct {
rawResponse *http.Response
@ -3687,6 +3787,22 @@ func (gr *GeoReplication) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e
return d.DecodeElement(gr2, &start)
}
// KeyInfo - Key information
type KeyInfo struct {
// Start - The date-time the key is active in ISO 8601 UTC time
Start string `xml:"Start"`
// Expiry - The date-time the key expires in ISO 8601 UTC time
Expiry string `xml:"Expiry"`
}
//NewKeyInfo creates a new KeyInfo struct with the correct time formatting & conversion
func NewKeyInfo(Start, Expiry time.Time) KeyInfo {
return KeyInfo{
Start: Start.UTC().Format(SASTimeFormat),
Expiry: Expiry.UTC().Format(SASTimeFormat),
}
}
// ListBlobsFlatSegmentResponse - An enumeration of blobs
type ListBlobsFlatSegmentResponse struct {
rawResponse *http.Response
@ -3694,10 +3810,10 @@ type ListBlobsFlatSegmentResponse struct {
XMLName xml.Name `xml:"EnumerationResults"`
ServiceEndpoint string `xml:"ServiceEndpoint,attr"`
ContainerName string `xml:"ContainerName,attr"`
Prefix string `xml:"Prefix"`
Marker string `xml:"Marker"`
MaxResults int32 `xml:"MaxResults"`
Delimiter string `xml:"Delimiter"`
Prefix *string `xml:"Prefix"`
Marker *string `xml:"Marker"`
MaxResults *int32 `xml:"MaxResults"`
Delimiter *string `xml:"Delimiter"`
Segment BlobFlatListSegment `xml:"Blobs"`
NextMarker Marker `xml:"NextMarker"`
}
@ -3757,10 +3873,10 @@ type ListBlobsHierarchySegmentResponse struct {
XMLName xml.Name `xml:"EnumerationResults"`
ServiceEndpoint string `xml:"ServiceEndpoint,attr"`
ContainerName string `xml:"ContainerName,attr"`
Prefix string `xml:"Prefix"`
Marker string `xml:"Marker"`
MaxResults int32 `xml:"MaxResults"`
Delimiter string `xml:"Delimiter"`
Prefix *string `xml:"Prefix"`
Marker *string `xml:"Marker"`
MaxResults *int32 `xml:"MaxResults"`
Delimiter *string `xml:"Delimiter"`
Segment BlobHierarchyListSegment `xml:"Blobs"`
NextMarker Marker `xml:"NextMarker"`
}
@ -3819,9 +3935,9 @@ type ListContainersSegmentResponse struct {
// XMLName is used for marshalling and is subject to removal in a future release.
XMLName xml.Name `xml:"EnumerationResults"`
ServiceEndpoint string `xml:"ServiceEndpoint,attr"`
Prefix string `xml:"Prefix"`
Prefix *string `xml:"Prefix"`
Marker *string `xml:"Marker"`
MaxResults int32 `xml:"MaxResults"`
MaxResults *int32 `xml:"MaxResults"`
ContainerItems []ContainerItem `xml:"Containers>Container"`
NextMarker Marker `xml:"NextMarker"`
}
@ -4854,7 +4970,91 @@ func (sss StorageServiceStats) Version() string {
return sss.rawResponse.Header.Get("x-ms-version")
}
// UserDelegationKey - A user delegation key
type UserDelegationKey struct {
rawResponse *http.Response
// SignedOid - The Azure Active Directory object ID in GUID format.
SignedOid string `xml:"SignedOid"`
// SignedTid - The Azure Active Directory tenant ID in GUID format
SignedTid string `xml:"SignedTid"`
// SignedStart - The date-time the key is active
SignedStart time.Time `xml:"SignedStart"`
// SignedExpiry - The date-time the key expires
SignedExpiry time.Time `xml:"SignedExpiry"`
// SignedService - Abbreviation of the Azure Storage service that accepts the key
SignedService string `xml:"SignedService"`
// SignedVersion - The service version that created the key
SignedVersion string `xml:"SignedVersion"`
// Value - The key as a base64 string
Value string `xml:"Value"`
}
func (udk UserDelegationKey) ComputeHMACSHA256(message string) (base64String string) {
bytes, _ := base64.StdEncoding.DecodeString(udk.Value)
h := hmac.New(sha256.New, bytes)
h.Write([]byte(message))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
// MarshalXML implements the xml.Marshaler interface for UserDelegationKey.
func (udk UserDelegationKey) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
udk2 := (*userDelegationKey)(unsafe.Pointer(&udk))
return e.EncodeElement(*udk2, start)
}
// UnmarshalXML implements the xml.Unmarshaler interface for UserDelegationKey.
func (udk *UserDelegationKey) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
udk2 := (*userDelegationKey)(unsafe.Pointer(udk))
return d.DecodeElement(udk2, &start)
}
// Response returns the raw HTTP response object.
func (udk UserDelegationKey) Response() *http.Response {
return udk.rawResponse
}
// StatusCode returns the HTTP status code of the response, e.g. 200.
func (udk UserDelegationKey) StatusCode() int {
return udk.rawResponse.StatusCode
}
// Status returns the HTTP status message of the response, e.g. "200 OK".
func (udk UserDelegationKey) Status() string {
return udk.rawResponse.Status
}
// Date returns the value for header Date.
func (udk UserDelegationKey) Date() time.Time {
s := udk.rawResponse.Header.Get("Date")
if s == "" {
return time.Time{}
}
t, err := time.Parse(time.RFC1123, s)
if err != nil {
t = time.Time{}
}
return t
}
// ErrorCode returns the value for header x-ms-error-code.
func (udk UserDelegationKey) ErrorCode() string {
return udk.rawResponse.Header.Get("x-ms-error-code")
}
// RequestID returns the value for header x-ms-request-id.
func (udk UserDelegationKey) RequestID() string {
return udk.rawResponse.Header.Get("x-ms-request-id")
}
// Version returns the value for header x-ms-version.
func (udk UserDelegationKey) Version() string {
return udk.rawResponse.Header.Get("x-ms-version")
}
func init() {
if reflect.TypeOf((*UserDelegationKey)(nil)).Elem().Size() != reflect.TypeOf((*userDelegationKey)(nil)).Elem().Size() {
validateError(errors.New("size mismatch between UserDelegationKey and userDelegationKey"))
}
if reflect.TypeOf((*AccessPolicy)(nil)).Elem().Size() != reflect.TypeOf((*accessPolicy)(nil)).Elem().Size() {
validateError(errors.New("size mismatch between AccessPolicy and accessPolicy"))
}
@ -4870,7 +5070,7 @@ func init() {
}
const (
rfc3339Format = "2006-01-02T15:04:05.0000000Z07:00"
rfc3339Format = "2006-01-02T15:04:05Z" //This was wrong in the generated code, FYI
)
// used to convert times from UTC to GMT before sending across the wire
@ -4928,6 +5128,18 @@ func (c *base64Encoded) UnmarshalText(data []byte) error {
return nil
}
// internal type used for marshalling
type userDelegationKey struct {
rawResponse *http.Response
SignedOid string `xml:"SignedOid"`
SignedTid string `xml:"SignedTid"`
SignedStart timeRFC3339 `xml:"SignedStart"`
SignedExpiry timeRFC3339 `xml:"SignedExpiry"`
SignedService string `xml:"SignedService"`
SignedVersion string `xml:"SignedVersion"`
Value string `xml:"Value"`
}
// internal type used for marshalling
type accessPolicy struct {
Start timeRFC3339 `xml:"Start"`

View file

@ -7,14 +7,13 @@ import (
"context"
"encoding/base64"
"encoding/xml"
"github.com/Azure/azure-pipeline-go/pipeline"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"time"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// pageBlobClient is the client for the PageBlob methods of the Azblob service.
@ -414,8 +413,8 @@ func (client pageBlobClient) getPageRangesResponder(resp pipeline.Response) (pip
return result, nil
}
// GetPageRangesDiff [Update] The Get Page Ranges Diff operation returns the list of valid page ranges for a page blob
// that were changed between target blob and previous snapshot.
// GetPageRangesDiff the Get Page Ranges Diff operation returns the list of valid page ranges for a page blob that were
// changed between target blob and previous snapshot.
//
// snapshot is the snapshot parameter is an opaque DateTime value that, when present, specifies the blob snapshot to
// retrieve. For more information on working with blob snapshots, see <a
@ -797,17 +796,21 @@ func (client pageBlobClient) uploadPagesResponder(resp pipeline.Response) (pipel
// specify this header value to operate only on a blob if it has been modified since the specified date/time.
// ifUnmodifiedSince is specify this header value to operate only on a blob if it has not been modified since the
// specified date/time. ifMatch is specify an ETag value to operate only on blobs with a matching value. ifNoneMatch is
// specify an ETag value to operate only on blobs without a matching value. requestID is provides a client-generated,
// opaque value with a 1 KB character limit that is recorded in the analytics logs when storage analytics logging is
// enabled.
func (client pageBlobClient) UploadPagesFromURL(ctx context.Context, sourceURL string, sourceRange string, contentLength int64, rangeParameter string, sourceContentMD5 []byte, timeout *int32, leaseID *string, ifSequenceNumberLessThanOrEqualTo *int64, ifSequenceNumberLessThan *int64, ifSequenceNumberEqualTo *int64, ifModifiedSince *time.Time, ifUnmodifiedSince *time.Time, ifMatch *ETag, ifNoneMatch *ETag, requestID *string) (*PageBlobUploadPagesFromURLResponse, error) {
// specify an ETag value to operate only on blobs without a matching value. sourceIfModifiedSince is specify this
// header value to operate only on a blob if it has been modified since the specified date/time.
// sourceIfUnmodifiedSince is specify this header value to operate only on a blob if it has not been modified since the
// specified date/time. sourceIfMatch is specify an ETag value to operate only on blobs with a matching value.
// sourceIfNoneMatch is specify an ETag value to operate only on blobs without a matching value. requestID is provides
// a client-generated, opaque value with a 1 KB character limit that is recorded in the analytics logs when storage
// analytics logging is enabled.
func (client pageBlobClient) UploadPagesFromURL(ctx context.Context, sourceURL string, sourceRange string, contentLength int64, rangeParameter string, sourceContentMD5 []byte, timeout *int32, leaseID *string, ifSequenceNumberLessThanOrEqualTo *int64, ifSequenceNumberLessThan *int64, ifSequenceNumberEqualTo *int64, ifModifiedSince *time.Time, ifUnmodifiedSince *time.Time, ifMatch *ETag, ifNoneMatch *ETag, sourceIfModifiedSince *time.Time, sourceIfUnmodifiedSince *time.Time, sourceIfMatch *ETag, sourceIfNoneMatch *ETag, requestID *string) (*PageBlobUploadPagesFromURLResponse, error) {
if err := validate([]validation{
{targetValue: timeout,
constraints: []constraint{{target: "timeout", name: null, rule: false,
chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 0, chain: nil}}}}}}); err != nil {
return nil, err
}
req, err := client.uploadPagesFromURLPreparer(sourceURL, sourceRange, contentLength, rangeParameter, sourceContentMD5, timeout, leaseID, ifSequenceNumberLessThanOrEqualTo, ifSequenceNumberLessThan, ifSequenceNumberEqualTo, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, requestID)
req, err := client.uploadPagesFromURLPreparer(sourceURL, sourceRange, contentLength, rangeParameter, sourceContentMD5, timeout, leaseID, ifSequenceNumberLessThanOrEqualTo, ifSequenceNumberLessThan, ifSequenceNumberEqualTo, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, sourceIfModifiedSince, sourceIfUnmodifiedSince, sourceIfMatch, sourceIfNoneMatch, requestID)
if err != nil {
return nil, err
}
@ -819,7 +822,7 @@ func (client pageBlobClient) UploadPagesFromURL(ctx context.Context, sourceURL s
}
// uploadPagesFromURLPreparer prepares the UploadPagesFromURL request.
func (client pageBlobClient) uploadPagesFromURLPreparer(sourceURL string, sourceRange string, contentLength int64, rangeParameter string, sourceContentMD5 []byte, timeout *int32, leaseID *string, ifSequenceNumberLessThanOrEqualTo *int64, ifSequenceNumberLessThan *int64, ifSequenceNumberEqualTo *int64, ifModifiedSince *time.Time, ifUnmodifiedSince *time.Time, ifMatch *ETag, ifNoneMatch *ETag, requestID *string) (pipeline.Request, error) {
func (client pageBlobClient) uploadPagesFromURLPreparer(sourceURL string, sourceRange string, contentLength int64, rangeParameter string, sourceContentMD5 []byte, timeout *int32, leaseID *string, ifSequenceNumberLessThanOrEqualTo *int64, ifSequenceNumberLessThan *int64, ifSequenceNumberEqualTo *int64, ifModifiedSince *time.Time, ifUnmodifiedSince *time.Time, ifMatch *ETag, ifNoneMatch *ETag, sourceIfModifiedSince *time.Time, sourceIfUnmodifiedSince *time.Time, sourceIfMatch *ETag, sourceIfNoneMatch *ETag, requestID *string) (pipeline.Request, error) {
req, err := pipeline.NewRequest("PUT", client.url, nil)
if err != nil {
return req, pipeline.NewError(err, "failed to create request")
@ -861,6 +864,18 @@ func (client pageBlobClient) uploadPagesFromURLPreparer(sourceURL string, source
if ifNoneMatch != nil {
req.Header.Set("If-None-Match", string(*ifNoneMatch))
}
if sourceIfModifiedSince != nil {
req.Header.Set("x-ms-source-if-modified-since", (*sourceIfModifiedSince).In(gmt).Format(time.RFC1123))
}
if sourceIfUnmodifiedSince != nil {
req.Header.Set("x-ms-source-if-unmodified-since", (*sourceIfUnmodifiedSince).In(gmt).Format(time.RFC1123))
}
if sourceIfMatch != nil {
req.Header.Set("x-ms-source-if-match", string(*sourceIfMatch))
}
if sourceIfNoneMatch != nil {
req.Header.Set("x-ms-source-if-none-match", string(*sourceIfNoneMatch))
}
req.Header.Set("x-ms-version", ServiceVersion)
if requestID != nil {
req.Header.Set("x-ms-client-request-id", *requestID)

View file

@ -7,9 +7,8 @@ import (
"bytes"
"context"
"encoding/xml"
"io/ioutil"
"github.com/Azure/azure-pipeline-go/pipeline"
"io/ioutil"
)
type responder func(resp pipeline.Response) (result pipeline.Response, err error)

View file

@ -6,10 +6,9 @@ package azblob
import (
"bytes"
"fmt"
"github.com/Azure/azure-pipeline-go/pipeline"
"net"
"net/http"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// if you want to provide custom error handling set this variable to your constructor function

View file

@ -7,13 +7,12 @@ import (
"bytes"
"context"
"encoding/xml"
"github.com/Azure/azure-pipeline-go/pipeline"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// serviceClient is the client for the Service methods of the Azblob service.
@ -204,6 +203,85 @@ func (client serviceClient) getStatisticsResponder(resp pipeline.Response) (pipe
return result, nil
}
// GetUserDelegationKey retrieves a user delgation key for the Blob service. This is only a valid operation when using
// bearer token authentication.
//
// timeout is the timeout parameter is expressed in seconds. For more information, see <a
// href="https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/setting-timeouts-for-blob-service-operations">Setting
// Timeouts for Blob Service Operations.</a> requestID is provides a client-generated, opaque value with a 1 KB
// character limit that is recorded in the analytics logs when storage analytics logging is enabled.
func (client serviceClient) GetUserDelegationKey(ctx context.Context, keyInfo KeyInfo, timeout *int32, requestID *string) (*UserDelegationKey, error) {
if err := validate([]validation{
{targetValue: timeout,
constraints: []constraint{{target: "timeout", name: null, rule: false,
chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 0, chain: nil}}}}}}); err != nil {
return nil, err
}
req, err := client.getUserDelegationKeyPreparer(keyInfo, timeout, requestID)
if err != nil {
return nil, err
}
resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.getUserDelegationKeyResponder}, req)
if err != nil {
return nil, err
}
return resp.(*UserDelegationKey), err
}
// getUserDelegationKeyPreparer prepares the GetUserDelegationKey request.
func (client serviceClient) getUserDelegationKeyPreparer(keyInfo KeyInfo, timeout *int32, requestID *string) (pipeline.Request, error) {
req, err := pipeline.NewRequest("POST", client.url, nil)
if err != nil {
return req, pipeline.NewError(err, "failed to create request")
}
params := req.URL.Query()
if timeout != nil {
params.Set("timeout", strconv.FormatInt(int64(*timeout), 10))
}
params.Set("restype", "service")
params.Set("comp", "userdelegationkey")
req.URL.RawQuery = params.Encode()
req.Header.Set("x-ms-version", ServiceVersion)
if requestID != nil {
req.Header.Set("x-ms-client-request-id", *requestID)
}
b, err := xml.Marshal(keyInfo)
if err != nil {
return req, pipeline.NewError(err, "failed to marshal request body")
}
req.Header.Set("Content-Type", "application/xml")
err = req.SetBody(bytes.NewReader(b))
if err != nil {
return req, pipeline.NewError(err, "failed to set request body")
}
return req, nil
}
// getUserDelegationKeyResponder handles the response to the GetUserDelegationKey request.
func (client serviceClient) getUserDelegationKeyResponder(resp pipeline.Response) (pipeline.Response, error) {
err := validateResponse(resp, http.StatusOK)
if resp == nil {
return nil, err
}
result := &UserDelegationKey{rawResponse: resp.Response()}
if err != nil {
return result, err
}
defer resp.Response().Body.Close()
b, err := ioutil.ReadAll(resp.Response().Body)
if err != nil {
return result, err
}
if len(b) > 0 {
b = removeBOM(b)
err = xml.Unmarshal(b, result)
if err != nil {
return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body")
}
}
return result, nil
}
// ListContainersSegment the List Containers Segment operation returns a list of the containers under the specified
// account
//

View file

@ -5,11 +5,10 @@ package azblob
import (
"fmt"
"github.com/Azure/azure-pipeline-go/pipeline"
"reflect"
"regexp"
"strings"
"github.com/Azure/azure-pipeline-go/pipeline"
)
// Constraint stores constraint name, target field name

View file

@ -208,7 +208,7 @@ func (e errorList) Error() string {
// How do we want to handle the array size being zero
if size := len(e); size > 0 {
for i := 0; i < size; i++ {
msg += fmt.Sprintf("%s", e[i].Error())
msg += e[i].Error()
// We check the next index to see if it is within the slice.
// If it is, then we append a newline. We do this, because unit tests
// could be broken with the additional '\n'

View file

@ -185,13 +185,12 @@ func ValuesAtPath(i interface{}, path string) ([]interface{}, error) {
// SetValueAtPath sets a value at the case insensitive lexical path inside
// of a structure.
func SetValueAtPath(i interface{}, path string, v interface{}) {
if rvals := rValuesAtPath(i, path, true, false, v == nil); rvals != nil {
for _, rval := range rvals {
if rval.Kind() == reflect.Ptr && rval.IsNil() {
continue
}
setValue(rval, v)
rvals := rValuesAtPath(i, path, true, false, v == nil)
for _, rval := range rvals {
if rval.Kind() == reflect.Ptr && rval.IsNil() {
continue
}
setValue(rval, v)
}
}

View file

@ -14,12 +14,12 @@ import (
// struct and override the specific methods. For example, to override only
// the MaxRetries method:
//
// type retryer struct {
// client.DefaultRetryer
// }
// type retryer struct {
// client.DefaultRetryer
// }
//
// // This implementation always has 100 max retries
// func (d retryer) MaxRetries() int { return 100 }
// // This implementation always has 100 max retries
// func (d retryer) MaxRetries() int { return 100 }
type DefaultRetryer struct {
NumMaxRetries int
}
@ -34,8 +34,8 @@ func (d DefaultRetryer) MaxRetries() int {
func (d DefaultRetryer) RetryRules(r *request.Request) time.Duration {
// Set the upper limit of delay in retrying at ~five minutes
minTime := 30
throttle := d.shouldThrottle(r)
if throttle {
isThrottle := r.IsErrorThrottle()
if isThrottle {
if delay, ok := getRetryDelay(r); ok {
return delay
}
@ -44,7 +44,7 @@ func (d DefaultRetryer) RetryRules(r *request.Request) time.Duration {
}
retryCount := r.RetryCount
if throttle && retryCount > 8 {
if isThrottle && retryCount > 8 {
retryCount = 8
} else if retryCount > 13 {
retryCount = 13
@ -65,21 +65,8 @@ func (d DefaultRetryer) ShouldRetry(r *request.Request) bool {
if r.HTTPResponse.StatusCode >= 500 && r.HTTPResponse.StatusCode != 501 {
return true
}
return r.IsErrorRetryable() || d.shouldThrottle(r)
}
// ShouldThrottle returns true if the request should be throttled.
func (d DefaultRetryer) shouldThrottle(r *request.Request) bool {
switch r.HTTPResponse.StatusCode {
case 429:
case 502:
case 503:
case 504:
default:
return r.IsErrorThrottle()
}
return true
return r.IsErrorRetryable() || r.IsErrorThrottle()
}
// This will look in the Retry-After header, RFC 7231, for how long

View file

@ -67,10 +67,14 @@ func logRequest(r *request.Request) {
if !bodySeekable {
r.SetReaderBody(aws.ReadSeekCloser(r.HTTPRequest.Body))
}
// Reset the request body because dumpRequest will re-wrap the r.HTTPRequest's
// Body as a NoOpCloser and will not be reset after read by the HTTP
// client reader.
r.ResetBody()
// Reset the request body because dumpRequest will re-wrap the
// r.HTTPRequest's Body as a NoOpCloser and will not be reset after
// read by the HTTP client reader.
if err := r.Error; err != nil {
r.Config.Logger.Log(fmt.Sprintf(logReqErrMsg,
r.ClientInfo.ServiceName, r.Operation.Name, err))
return
}
}
r.Config.Logger.Log(fmt.Sprintf(logReqMsg,

View file

@ -20,7 +20,7 @@ type RequestRetryer interface{}
// A Config provides service configuration for service clients. By default,
// all clients will use the defaults.DefaultConfig structure.
//
// // Create Session with MaxRetry configuration to be shared by multiple
// // Create Session with MaxRetries configuration to be shared by multiple
// // service clients.
// sess := session.Must(session.NewSession(&aws.Config{
// MaxRetries: aws.Int(3),
@ -251,7 +251,7 @@ type Config struct {
// NewConfig returns a new Config pointer that can be chained with builder
// methods to set multiple configuration values inline without using pointers.
//
// // Create Session with MaxRetry configuration to be shared by multiple
// // Create Session with MaxRetries configuration to be shared by multiple
// // service clients.
// sess := session.Must(session.NewSession(aws.NewConfig().
// WithMaxRetries(3),

View file

@ -159,9 +159,9 @@ func handleSendError(r *request.Request, err error) {
Body: ioutil.NopCloser(bytes.NewReader([]byte{})),
}
}
// Catch all other request errors.
// Catch all request errors, and let the default retrier determine
// if the error is retryable.
r.Error = awserr.New("RequestError", "send request failed", err)
r.Retryable = aws.Bool(true) // network errors are retryable
// Override the error with a context canceled error, if that was canceled.
ctx := r.Context()
@ -184,37 +184,39 @@ var ValidateResponseHandler = request.NamedHandler{Name: "core.ValidateResponseH
// AfterRetryHandler performs final checks to determine if the request should
// be retried and how long to delay.
var AfterRetryHandler = request.NamedHandler{Name: "core.AfterRetryHandler", Fn: func(r *request.Request) {
// If one of the other handlers already set the retry state
// we don't want to override it based on the service's state
if r.Retryable == nil || aws.BoolValue(r.Config.EnforceShouldRetryCheck) {
r.Retryable = aws.Bool(r.ShouldRetry(r))
}
if r.WillRetry() {
r.RetryDelay = r.RetryRules(r)
if sleepFn := r.Config.SleepDelay; sleepFn != nil {
// Support SleepDelay for backwards compatibility and testing
sleepFn(r.RetryDelay)
} else if err := aws.SleepWithContext(r.Context(), r.RetryDelay); err != nil {
r.Error = awserr.New(request.CanceledErrorCode,
"request context canceled", err)
r.Retryable = aws.Bool(false)
return
var AfterRetryHandler = request.NamedHandler{
Name: "core.AfterRetryHandler",
Fn: func(r *request.Request) {
// If one of the other handlers already set the retry state
// we don't want to override it based on the service's state
if r.Retryable == nil || aws.BoolValue(r.Config.EnforceShouldRetryCheck) {
r.Retryable = aws.Bool(r.ShouldRetry(r))
}
// when the expired token exception occurs the credentials
// need to be expired locally so that the next request to
// get credentials will trigger a credentials refresh.
if r.IsErrorExpired() {
r.Config.Credentials.Expire()
}
if r.WillRetry() {
r.RetryDelay = r.RetryRules(r)
r.RetryCount++
r.Error = nil
}
}}
if sleepFn := r.Config.SleepDelay; sleepFn != nil {
// Support SleepDelay for backwards compatibility and testing
sleepFn(r.RetryDelay)
} else if err := aws.SleepWithContext(r.Context(), r.RetryDelay); err != nil {
r.Error = awserr.New(request.CanceledErrorCode,
"request context canceled", err)
r.Retryable = aws.Bool(false)
return
}
// when the expired token exception occurs the credentials
// need to be expired locally so that the next request to
// get credentials will trigger a credentials refresh.
if r.IsErrorExpired() {
r.Config.Credentials.Expire()
}
r.RetryCount++
r.Error = nil
}
}}
// ValidateEndpointHandler is a request handler to validate a request had the
// appropriate Region and Endpoint set. Will set r.Error if the endpoint or

View file

@ -84,6 +84,12 @@ type Value struct {
ProviderName string
}
// HasKeys returns if the credentials Value has both AccessKeyID and
// SecretAccessKey value set.
func (v Value) HasKeys() bool {
return len(v.AccessKeyID) != 0 && len(v.SecretAccessKey) != 0
}
// A Provider is the interface for any component which will provide credentials
// Value. A provider is required to manage its own Expired state, and what to
// be expired means.

View file

@ -98,8 +98,8 @@ func NewProviderClient(cfg aws.Config, handlers request.Handlers, endpoint strin
return p
}
// NewCredentialsClient returns a Credentials wrapper for retrieving credentials
// from an arbitrary endpoint concurrently. The client will request the
// NewCredentialsClient returns a pointer to a new Credentials object
// wrapping the endpoint credentials Provider.
func NewCredentialsClient(cfg aws.Config, handlers request.Handlers, endpoint string, options ...func(*Provider)) *credentials.Credentials {
return credentials.NewCredentials(NewProviderClient(cfg, handlers, endpoint, options...))
}

View file

@ -0,0 +1,100 @@
package stscreds
import (
"fmt"
"io/ioutil"
"strconv"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/aws/aws-sdk-go/service/sts/stsiface"
)
const (
// ErrCodeWebIdentity will be used as an error code when constructing
// a new error to be returned during session creation or retrieval.
ErrCodeWebIdentity = "WebIdentityErr"
// WebIdentityProviderName is the web identity provider name
WebIdentityProviderName = "WebIdentityCredentials"
)
// now is used to return a time.Time object representing
// the current time. This can be used to easily test and
// compare test values.
var now = time.Now
// WebIdentityRoleProvider is used to retrieve credentials using
// an OIDC token.
type WebIdentityRoleProvider struct {
credentials.Expiry
client stsiface.STSAPI
ExpiryWindow time.Duration
tokenFilePath string
roleARN string
roleSessionName string
}
// NewWebIdentityCredentials will return a new set of credentials with a given
// configuration, role arn, and token file path.
func NewWebIdentityCredentials(c client.ConfigProvider, roleARN, roleSessionName, path string) *credentials.Credentials {
svc := sts.New(c)
p := NewWebIdentityRoleProvider(svc, roleARN, roleSessionName, path)
return credentials.NewCredentials(p)
}
// NewWebIdentityRoleProvider will return a new WebIdentityRoleProvider with the
// provided stsiface.STSAPI
func NewWebIdentityRoleProvider(svc stsiface.STSAPI, roleARN, roleSessionName, path string) *WebIdentityRoleProvider {
return &WebIdentityRoleProvider{
client: svc,
tokenFilePath: path,
roleARN: roleARN,
roleSessionName: roleSessionName,
}
}
// Retrieve will attempt to assume a role from a token which is located at
// 'WebIdentityTokenFilePath' specified destination and if that is empty an
// error will be returned.
func (p *WebIdentityRoleProvider) Retrieve() (credentials.Value, error) {
b, err := ioutil.ReadFile(p.tokenFilePath)
if err != nil {
errMsg := fmt.Sprintf("unable to read file at %s", p.tokenFilePath)
return credentials.Value{}, awserr.New(ErrCodeWebIdentity, errMsg, err)
}
sessionName := p.roleSessionName
if len(sessionName) == 0 {
// session name is used to uniquely identify a session. This simply
// uses unix time in nanoseconds to uniquely identify sessions.
sessionName = strconv.FormatInt(now().UnixNano(), 10)
}
req, resp := p.client.AssumeRoleWithWebIdentityRequest(&sts.AssumeRoleWithWebIdentityInput{
RoleArn: &p.roleARN,
RoleSessionName: &sessionName,
WebIdentityToken: aws.String(string(b)),
})
// InvalidIdentityToken error is a temporary error that can occur
// when assuming an Role with a JWT web identity token.
req.RetryErrorCodes = append(req.RetryErrorCodes, sts.ErrCodeInvalidIdentityTokenException)
if err := req.Send(); err != nil {
return credentials.Value{}, awserr.New(ErrCodeWebIdentity, "failed to retrieve credentials", err)
}
p.SetExpiration(aws.TimeValue(resp.Credentials.Expiration), p.ExpiryWindow)
value := credentials.Value{
AccessKeyID: aws.StringValue(resp.Credentials.AccessKeyId),
SecretAccessKey: aws.StringValue(resp.Credentials.SecretAccessKey),
SessionToken: aws.StringValue(resp.Credentials.SessionToken),
ProviderName: WebIdentityProviderName,
}
return value, nil
}

View file

@ -1,30 +1,61 @@
// Package csm provides Client Side Monitoring (CSM) which enables sending metrics
// via UDP connection. Using the Start function will enable the reporting of
// metrics on a given port. If Start is called, with different parameters, again,
// a panic will occur.
// Package csm provides the Client Side Monitoring (CSM) client which enables
// sending metrics via UDP connection to the CSM agent. This package provides
// control options, and configuration for the CSM client. The client can be
// controlled manually, or automatically via the SDK's Session configuration.
//
// Pause can be called to pause any metrics publishing on a given port. Sessions
// that have had their handlers modified via InjectHandlers may still be used.
// However, the handlers will act as a no-op meaning no metrics will be published.
// Enabling CSM client via SDK's Session configuration
//
// The CSM client can be enabled automatically via SDK's Session configuration.
// The SDK's session configuration enables the CSM client if the AWS_CSM_PORT
// environment variable is set to a non-empty value.
//
// The configuration options for the CSM client via the SDK's session
// configuration are:
//
// * AWS_CSM_PORT=<port number>
// The port number the CSM agent will receive metrics on.
//
// * AWS_CSM_HOST=<hostname or ip>
// The hostname, or IP address the CSM agent will receive metrics on.
// Without port number.
//
// Manually enabling the CSM client
//
// The CSM client can be started, paused, and resumed manually. The Start
// function will enable the CSM client to publish metrics to the CSM agent. It
// is safe to call Start concurrently, but if Start is called additional times
// with different ClientID or address it will panic.
//
// Example:
// r, err := csm.Start("clientID", ":31000")
// if err != nil {
// panic(fmt.Errorf("failed starting CSM: %v", err))
// }
//
// When controlling the CSM client manually, you must also inject its request
// handlers into the SDK's Session configuration for the SDK's API clients to
// publish metrics.
//
// sess, err := session.NewSession(&aws.Config{})
// if err != nil {
// panic(fmt.Errorf("failed loading session: %v", err))
// }
//
// // Add CSM client's metric publishing request handlers to the SDK's
// // Session Configuration.
// r.InjectHandlers(&sess.Handlers)
//
// client := s3.New(sess)
// resp, err := client.GetObject(&s3.GetObjectInput{
// Bucket: aws.String("bucket"),
// Key: aws.String("key"),
// })
// Controlling CSM client
//
// Once the CSM client has been enabled the Get function will return a Reporter
// value that you can use to pause and resume the metrics published to the CSM
// agent. If Get function is called before the reporter is enabled with the
// Start function or via SDK's Session configuration nil will be returned.
//
// The Pause method can be called to stop the CSM client publishing metrics to
// the CSM agent. The Continue method will resume metric publishing.
//
// // Get the CSM client Reporter.
// r := csm.Get()
//
// // Will pause monitoring
// r.Pause()
@ -35,12 +66,4 @@
//
// // Resume monitoring
// r.Continue()
//
// Start returns a Reporter that is used to enable or disable monitoring. If
// access to the Reporter is required later, calling Get will return the Reporter
// singleton.
//
// Example:
// r := csm.Get()
// r.Continue()
package csm

View file

@ -2,6 +2,7 @@ package csm
import (
"fmt"
"strings"
"sync"
)
@ -9,19 +10,40 @@ var (
lock sync.Mutex
)
// Client side metric handler names
const (
APICallMetricHandlerName = "awscsm.SendAPICallMetric"
APICallAttemptMetricHandlerName = "awscsm.SendAPICallAttemptMetric"
// DefaultPort is used when no port is specified.
DefaultPort = "31000"
// DefaultHost is the host that will be used when none is specified.
DefaultHost = "127.0.0.1"
)
// Start will start the a long running go routine to capture
// AddressWithDefaults returns a CSM address built from the host and port
// values. If the host or port is not set, default values will be used
// instead. If host is "localhost" it will be replaced with "127.0.0.1".
func AddressWithDefaults(host, port string) string {
if len(host) == 0 || strings.EqualFold(host, "localhost") {
host = DefaultHost
}
if len(port) == 0 {
port = DefaultPort
}
// Only IP6 host can contain a colon
if strings.Contains(host, ":") {
return "[" + host + "]:" + port
}
return host + ":" + port
}
// Start will start a long running go routine to capture
// client side metrics. Calling start multiple time will only
// start the metric listener once and will panic if a different
// client ID or port is passed in.
//
// Example:
// r, err := csm.Start("clientID", "127.0.0.1:8094")
// r, err := csm.Start("clientID", "127.0.0.1:31000")
// if err != nil {
// panic(fmt.Errorf("expected no error, but received %v", err))
// }

View file

@ -10,11 +10,6 @@ import (
"github.com/aws/aws-sdk-go/aws/request"
)
const (
// DefaultPort is used when no port is specified
DefaultPort = "31000"
)
// Reporter will gather metrics of API requests made and
// send those metrics to the CSM endpoint.
type Reporter struct {
@ -123,7 +118,7 @@ func (rep *Reporter) sendAPICallMetric(r *request.Request) {
Type: aws.String("ApiCall"),
AttemptCount: aws.Int(r.RetryCount + 1),
Region: r.Config.Region,
Latency: aws.Int(int(time.Now().Sub(r.Time) / time.Millisecond)),
Latency: aws.Int(int(time.Since(r.Time) / time.Millisecond)),
XAmzRequestID: aws.String(r.RequestID),
MaxRetriesExceeded: aws.Int(boolIntValue(r.RetryCount >= r.MaxRetries())),
}
@ -190,8 +185,9 @@ func (rep *Reporter) start() {
}
}
// Pause will pause the metric channel preventing any new metrics from
// being added.
// Pause will pause the metric channel preventing any new metrics from being
// added. It is safe to call concurrently with other calls to Pause, but if
// called concurently with Continue can lead to unexpected state.
func (rep *Reporter) Pause() {
lock.Lock()
defer lock.Unlock()
@ -203,8 +199,9 @@ func (rep *Reporter) Pause() {
rep.close()
}
// Continue will reopen the metric channel and allow for monitoring
// to be resumed.
// Continue will reopen the metric channel and allow for monitoring to be
// resumed. It is safe to call concurrently with other calls to Continue, but
// if called concurently with Pause can lead to unexpected state.
func (rep *Reporter) Continue() {
lock.Lock()
defer lock.Unlock()
@ -219,10 +216,18 @@ func (rep *Reporter) Continue() {
rep.metricsCh.Continue()
}
// Client side metric handler names
const (
APICallMetricHandlerName = "awscsm.SendAPICallMetric"
APICallAttemptMetricHandlerName = "awscsm.SendAPICallAttemptMetric"
)
// InjectHandlers will will enable client side metrics and inject the proper
// handlers to handle how metrics are sent.
//
// Example:
// InjectHandlers is NOT safe to call concurrently. Calling InjectHandlers
// multiple times may lead to unexpected behavior, (e.g. duplicate metrics).
//
// // Start must be called in order to inject the correct handlers
// r, err := csm.Start("clientID", "127.0.0.1:8094")
// if err != nil {

View file

@ -123,7 +123,7 @@ func unmarshalHandler(r *request.Request) {
defer r.HTTPResponse.Body.Close()
b := &bytes.Buffer{}
if _, err := io.Copy(b, r.HTTPResponse.Body); err != nil {
r.Error = awserr.New(request.ErrCodeSerialization, "unable to unmarshal EC2 metadata respose", err)
r.Error = awserr.New(request.ErrCodeSerialization, "unable to unmarshal EC2 metadata response", err)
return
}
@ -136,7 +136,7 @@ func unmarshalError(r *request.Request) {
defer r.HTTPResponse.Body.Close()
b := &bytes.Buffer{}
if _, err := io.Copy(b, r.HTTPResponse.Body); err != nil {
r.Error = awserr.New(request.ErrCodeSerialization, "unable to unmarshal EC2 metadata error respose", err)
r.Error = awserr.New(request.ErrCodeSerialization, "unable to unmarshal EC2 metadata error response", err)
return
}

File diff suppressed because it is too large Load diff

View file

@ -170,10 +170,13 @@ func PartitionForRegion(ps []Partition, regionID string) (Partition, bool) {
// A Partition provides the ability to enumerate the partition's regions
// and services.
type Partition struct {
id string
p *partition
id, dnsSuffix string
p *partition
}
// DNSSuffix returns the base domain name of the partition.
func (p Partition) DNSSuffix() string { return p.dnsSuffix }
// ID returns the identifier of the partition.
func (p Partition) ID() string { return p.id }

View file

@ -54,8 +54,9 @@ type partition struct {
func (p partition) Partition() Partition {
return Partition{
id: p.ID,
p: &p,
dnsSuffix: p.DNSSuffix,
id: p.ID,
p: &p,
}
}

View file

@ -15,12 +15,15 @@ type offsetReader struct {
closed bool
}
func newOffsetReader(buf io.ReadSeeker, offset int64) *offsetReader {
func newOffsetReader(buf io.ReadSeeker, offset int64) (*offsetReader, error) {
reader := &offsetReader{}
buf.Seek(offset, sdkio.SeekStart)
_, err := buf.Seek(offset, sdkio.SeekStart)
if err != nil {
return nil, err
}
reader.buf = buf
return reader
return reader, nil
}
// Close will close the instance of the offset reader's access to
@ -54,7 +57,9 @@ func (o *offsetReader) Seek(offset int64, whence int) (int64, error) {
// CloseAndCopy will return a new offsetReader with a copy of the old buffer
// and close the old buffer.
func (o *offsetReader) CloseAndCopy(offset int64) *offsetReader {
o.Close()
func (o *offsetReader) CloseAndCopy(offset int64) (*offsetReader, error) {
if err := o.Close(); err != nil {
return nil, err
}
return newOffsetReader(o.buf, offset)
}

View file

@ -64,6 +64,15 @@ type Request struct {
LastSignedAt time.Time
DisableFollowRedirects bool
// Additional API error codes that should be retried. IsErrorRetryable
// will consider these codes in addition to its built in cases.
RetryErrorCodes []string
// Additional API error codes that should be retried with throttle backoff
// delay. IsErrorThrottle will consider these codes in addition to its
// built in cases.
ThrottleErrorCodes []string
// A value greater than 0 instructs the request to be signed as Presigned URL
// You should not set this field directly. Instead use Request's
// Presign or PresignRequest methods.
@ -263,7 +272,18 @@ func (r *Request) SetStringBody(s string) {
// SetReaderBody will set the request's body reader.
func (r *Request) SetReaderBody(reader io.ReadSeeker) {
r.Body = reader
r.BodyStart, _ = reader.Seek(0, sdkio.SeekCurrent) // Get the Bodies current offset.
if aws.IsReaderSeekable(reader) {
var err error
// Get the Bodies current offset so retries will start from the same
// initial position.
r.BodyStart, err = reader.Seek(0, sdkio.SeekCurrent)
if err != nil {
r.Error = awserr.New(ErrCodeSerialization,
"failed to determine start of request body", err)
return
}
}
r.ResetBody()
}
@ -335,9 +355,7 @@ func getPresignedURL(r *Request, expire time.Duration) (string, http.Header, err
}
const (
willRetry = "will retry"
notRetrying = "not retrying"
retryCount = "retry %v/%v"
)
func debugLogReqError(r *Request, stage, retryStr string, err error) {
@ -392,12 +410,16 @@ func (r *Request) Sign() error {
return r.Error
}
func (r *Request) getNextRequestBody() (io.ReadCloser, error) {
func (r *Request) getNextRequestBody() (body io.ReadCloser, err error) {
if r.safeBody != nil {
r.safeBody.Close()
}
r.safeBody = newOffsetReader(r.Body, r.BodyStart)
r.safeBody, err = newOffsetReader(r.Body, r.BodyStart)
if err != nil {
return nil, awserr.New(ErrCodeSerialization,
"failed to get next request body reader", err)
}
// Go 1.8 tightened and clarified the rules code needs to use when building
// requests with the http package. Go 1.8 removed the automatic detection
@ -414,10 +436,10 @@ func (r *Request) getNextRequestBody() (io.ReadCloser, error) {
// Related golang/go#18257
l, err := aws.SeekerLen(r.Body)
if err != nil {
return nil, awserr.New(ErrCodeSerialization, "failed to compute request body size", err)
return nil, awserr.New(ErrCodeSerialization,
"failed to compute request body size", err)
}
var body io.ReadCloser
if l == 0 {
body = NoBody
} else if l > 0 {
@ -484,23 +506,22 @@ func (r *Request) Send() error {
if err := r.sendRequest(); err == nil {
return nil
} else if !shouldRetryCancel(r.Error) {
}
r.Handlers.Retry.Run(r)
r.Handlers.AfterRetry.Run(r)
if r.Error != nil || !aws.BoolValue(r.Retryable) {
return r.Error
}
if err := r.prepareRetry(); err != nil {
r.Error = err
return err
} else {
r.Handlers.Retry.Run(r)
r.Handlers.AfterRetry.Run(r)
if r.Error != nil || !aws.BoolValue(r.Retryable) {
return r.Error
}
r.prepareRetry()
continue
}
}
}
func (r *Request) prepareRetry() {
func (r *Request) prepareRetry() error {
if r.Config.LogLevel.Matches(aws.LogDebugWithRequestRetries) {
r.Config.Logger.Log(fmt.Sprintf("DEBUG: Retrying Request %s/%s, attempt %d",
r.ClientInfo.ServiceName, r.Operation.Name, r.RetryCount))
@ -511,12 +532,19 @@ func (r *Request) prepareRetry() {
// the request's body even though the Client's Do returned.
r.HTTPRequest = copyHTTPRequest(r.HTTPRequest, nil)
r.ResetBody()
if err := r.Error; err != nil {
return awserr.New(ErrCodeSerialization,
"failed to prepare body for retry", err)
}
// Closing response body to ensure that no response body is leaked
// between retry attempts.
if r.HTTPResponse != nil && r.HTTPResponse.Body != nil {
r.HTTPResponse.Body.Close()
}
return nil
}
func (r *Request) sendRequest() (sendErr error) {
@ -572,48 +600,6 @@ func AddToUserAgent(r *Request, s string) {
r.HTTPRequest.Header.Set("User-Agent", s)
}
type temporary interface {
Temporary() bool
}
func shouldRetryCancel(origErr error) bool {
switch err := origErr.(type) {
case awserr.Error:
if err.Code() == CanceledErrorCode {
return false
}
return shouldRetryCancel(err.OrigErr())
case *url.Error:
if strings.Contains(err.Error(), "connection refused") {
// Refused connections should be retried as the service may not yet
// be running on the port. Go TCP dial considers refused
// connections as not temporary.
return true
}
// *url.Error only implements Temporary after golang 1.6 but since
// url.Error only wraps the error:
return shouldRetryCancel(err.Err)
case temporary:
// If the error is temporary, we want to allow continuation of the
// retry process
return err.Temporary() || isErrConnectionReset(origErr)
case nil:
// `awserr.Error.OrigErr()` can be nil, meaning there was an error but
// because we don't know the cause, it is marked as retryable. See
// TestRequest4xxUnretryable for an example.
return true
default:
switch err.Error() {
case "net/http: request canceled",
"net/http: request canceled while waiting for connection":
// known 1.5 error case when an http request is cancelled
return false
}
// here we don't know the error; so we allow a retry.
return true
}
}
// SanitizeHostForHeader removes default port from host and updates request.Host
func SanitizeHostForHeader(r *http.Request) {
host := getHost(r)

View file

@ -4,6 +4,8 @@ package request
import (
"net/http"
"github.com/aws/aws-sdk-go/aws/awserr"
)
// NoBody is a http.NoBody reader instructing Go HTTP client to not include
@ -24,7 +26,8 @@ var NoBody = http.NoBody
func (r *Request) ResetBody() {
body, err := r.getNextRequestBody()
if err != nil {
r.Error = err
r.Error = awserr.New(ErrCodeSerialization,
"failed to reset request body", err)
return
}

View file

@ -146,7 +146,7 @@ func (r *Request) nextPageTokens() []interface{} {
return nil
}
case bool:
if v == false {
if !v {
return nil
}
}

View file

@ -1,23 +1,41 @@
package request
import (
"net"
"net/url"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
)
// Retryer is an interface to control retry logic for a given service.
// The default implementation used by most services is the client.DefaultRetryer
// structure, which contains basic retry logic using exponential backoff.
// Retryer provides the interface drive the SDK's request retry behavior. The
// Retryer implementation is responsible for implementing exponential backoff,
// and determine if a request API error should be retried.
//
// client.DefaultRetryer is the SDK's default implementation of the Retryer. It
// uses the which uses the Request.IsErrorRetryable and Request.IsErrorThrottle
// methods to determine if the request is retried.
type Retryer interface {
// RetryRules return the retry delay that should be used by the SDK before
// making another request attempt for the failed request.
RetryRules(*Request) time.Duration
// ShouldRetry returns if the failed request is retryable.
//
// Implementations may consider request attempt count when determining if a
// request is retryable, but the SDK will use MaxRetries to limit the
// number of attempts a request are made.
ShouldRetry(*Request) bool
// MaxRetries is the number of times a request may be retried before
// failing.
MaxRetries() int
}
// WithRetryer sets a config Retryer value to the given Config returning it
// for chaining.
// WithRetryer sets a Retryer value to the given Config returning the Config
// value for chaining.
func WithRetryer(cfg *aws.Config, retryer Retryer) *aws.Config {
cfg.Retryer = retryer
return cfg
@ -108,32 +126,90 @@ func isNestedErrorRetryable(parentErr awserr.Error) bool {
// IsErrorRetryable returns whether the error is retryable, based on its Code.
// Returns false if error is nil.
func IsErrorRetryable(err error) bool {
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
return isCodeRetryable(aerr.Code()) || isNestedErrorRetryable(aerr)
}
if err == nil {
return false
}
return shouldRetryError(err)
}
type temporary interface {
Temporary() bool
}
func shouldRetryError(origErr error) bool {
switch err := origErr.(type) {
case awserr.Error:
if err.Code() == CanceledErrorCode {
return false
}
if isNestedErrorRetryable(err) {
return true
}
origErr := err.OrigErr()
var shouldRetry bool
if origErr != nil {
shouldRetry := shouldRetryError(origErr)
if err.Code() == "RequestError" && !shouldRetry {
return false
}
}
if isCodeRetryable(err.Code()) {
return true
}
return shouldRetry
case *url.Error:
if strings.Contains(err.Error(), "connection refused") {
// Refused connections should be retried as the service may not yet
// be running on the port. Go TCP dial considers refused
// connections as not temporary.
return true
}
// *url.Error only implements Temporary after golang 1.6 but since
// url.Error only wraps the error:
return shouldRetryError(err.Err)
case temporary:
if netErr, ok := err.(*net.OpError); ok && netErr.Op == "dial" {
return true
}
// If the error is temporary, we want to allow continuation of the
// retry process
return err.Temporary() || isErrConnectionReset(origErr)
case nil:
// `awserr.Error.OrigErr()` can be nil, meaning there was an error but
// because we don't know the cause, it is marked as retryable. See
// TestRequest4xxUnretryable for an example.
return true
default:
switch err.Error() {
case "net/http: request canceled",
"net/http: request canceled while waiting for connection":
// known 1.5 error case when an http request is cancelled
return false
}
// here we don't know the error; so we allow a retry.
return true
}
return false
}
// IsErrorThrottle returns whether the error is to be throttled based on its code.
// Returns false if error is nil.
func IsErrorThrottle(err error) bool {
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
return isCodeThrottle(aerr.Code())
}
if aerr, ok := err.(awserr.Error); ok && aerr != nil {
return isCodeThrottle(aerr.Code())
}
return false
}
// IsErrorExpiredCreds returns whether the error code is a credential expiry error.
// Returns false if error is nil.
// IsErrorExpiredCreds returns whether the error code is a credential expiry
// error. Returns false if error is nil.
func IsErrorExpiredCreds(err error) bool {
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
return isCodeExpiredCreds(aerr.Code())
}
if aerr, ok := err.(awserr.Error); ok && aerr != nil {
return isCodeExpiredCreds(aerr.Code())
}
return false
}
@ -143,17 +219,44 @@ func IsErrorExpiredCreds(err error) bool {
//
// Alias for the utility function IsErrorRetryable
func (r *Request) IsErrorRetryable() bool {
if isErrCode(r.Error, r.RetryErrorCodes) {
return true
}
return IsErrorRetryable(r.Error)
}
// IsErrorThrottle returns whether the error is to be throttled based on its code.
// Returns false if the request has no Error set
// IsErrorThrottle returns whether the error is to be throttled based on its
// code. Returns false if the request has no Error set.
//
// Alias for the utility function IsErrorThrottle
func (r *Request) IsErrorThrottle() bool {
if isErrCode(r.Error, r.ThrottleErrorCodes) {
return true
}
if r.HTTPResponse != nil {
switch r.HTTPResponse.StatusCode {
case 429, 502, 503, 504:
return true
}
}
return IsErrorThrottle(r.Error)
}
func isErrCode(err error, codes []string) bool {
if aerr, ok := err.(awserr.Error); ok && aerr != nil {
for _, code := range codes {
if code == aerr.Code() {
return true
}
}
}
return false
}
// IsErrorExpired returns whether the error code is a credential expiry error.
// Returns false if the request has no Error set.
//

View file

@ -14,6 +14,142 @@ import (
"github.com/aws/aws-sdk-go/internal/shareddefaults"
)
func resolveCredentials(cfg *aws.Config,
envCfg envConfig, sharedCfg sharedConfig,
handlers request.Handlers,
sessOpts Options,
) (*credentials.Credentials, error) {
switch {
case len(sessOpts.Profile) != 0:
// User explicitly provided an Profile in the session's configuration
// so load that profile from shared config first.
// Github(aws/aws-sdk-go#2727)
return resolveCredsFromProfile(cfg, envCfg, sharedCfg, handlers, sessOpts)
case envCfg.Creds.HasKeys():
// Environment credentials
return credentials.NewStaticCredentialsFromCreds(envCfg.Creds), nil
case len(envCfg.WebIdentityTokenFilePath) != 0:
// Web identity token from environment, RoleARN required to also be
// set.
return assumeWebIdentity(cfg, handlers,
envCfg.WebIdentityTokenFilePath,
envCfg.RoleARN,
envCfg.RoleSessionName,
)
default:
// Fallback to the "default" credential resolution chain.
return resolveCredsFromProfile(cfg, envCfg, sharedCfg, handlers, sessOpts)
}
}
// WebIdentityEmptyRoleARNErr will occur if 'AWS_WEB_IDENTITY_TOKEN_FILE' was set but
// 'AWS_IAM_ROLE_ARN' was not set.
var WebIdentityEmptyRoleARNErr = awserr.New(stscreds.ErrCodeWebIdentity, "role ARN is not set", nil)
// WebIdentityEmptyTokenFilePathErr will occur if 'AWS_IAM_ROLE_ARN' was set but
// 'AWS_WEB_IDENTITY_TOKEN_FILE' was not set.
var WebIdentityEmptyTokenFilePathErr = awserr.New(stscreds.ErrCodeWebIdentity, "token file path is not set", nil)
func assumeWebIdentity(cfg *aws.Config, handlers request.Handlers,
filepath string,
roleARN, sessionName string,
) (*credentials.Credentials, error) {
if len(filepath) == 0 {
return nil, WebIdentityEmptyTokenFilePathErr
}
if len(roleARN) == 0 {
return nil, WebIdentityEmptyRoleARNErr
}
creds := stscreds.NewWebIdentityCredentials(
&Session{
Config: cfg,
Handlers: handlers.Copy(),
},
roleARN,
sessionName,
filepath,
)
return creds, nil
}
func resolveCredsFromProfile(cfg *aws.Config,
envCfg envConfig, sharedCfg sharedConfig,
handlers request.Handlers,
sessOpts Options,
) (creds *credentials.Credentials, err error) {
switch {
case sharedCfg.SourceProfile != nil:
// Assume IAM role with credentials source from a different profile.
creds, err = resolveCredsFromProfile(cfg, envCfg,
*sharedCfg.SourceProfile, handlers, sessOpts,
)
case sharedCfg.Creds.HasKeys():
// Static Credentials from Shared Config/Credentials file.
creds = credentials.NewStaticCredentialsFromCreds(
sharedCfg.Creds,
)
case len(sharedCfg.CredentialProcess) != 0:
// Get credentials from CredentialProcess
creds = processcreds.NewCredentials(sharedCfg.CredentialProcess)
case len(sharedCfg.CredentialSource) != 0:
creds, err = resolveCredsFromSource(cfg, envCfg,
sharedCfg, handlers, sessOpts,
)
case len(sharedCfg.WebIdentityTokenFile) != 0:
// Credentials from Assume Web Identity token require an IAM Role, and
// that roll will be assumed. May be wrapped with another assume role
// via SourceProfile.
return assumeWebIdentity(cfg, handlers,
sharedCfg.WebIdentityTokenFile,
sharedCfg.RoleARN,
sharedCfg.RoleSessionName,
)
default:
// Fallback to default credentials provider, include mock errors for
// the credential chain so user can identify why credentials failed to
// be retrieved.
creds = credentials.NewCredentials(&credentials.ChainProvider{
VerboseErrors: aws.BoolValue(cfg.CredentialsChainVerboseErrors),
Providers: []credentials.Provider{
&credProviderError{
Err: awserr.New("EnvAccessKeyNotFound",
"failed to find credentials in the environment.", nil),
},
&credProviderError{
Err: awserr.New("SharedCredsLoad",
fmt.Sprintf("failed to load profile, %s.", envCfg.Profile), nil),
},
defaults.RemoteCredProvider(*cfg, handlers),
},
})
}
if err != nil {
return nil, err
}
if len(sharedCfg.RoleARN) > 0 {
cfgCp := *cfg
cfgCp.Credentials = creds
return credsFromAssumeRole(cfgCp, handlers, sharedCfg, sessOpts)
}
return creds, nil
}
// valid credential source values
const (
credSourceEc2Metadata = "Ec2InstanceMetadata"
@ -21,112 +157,33 @@ const (
credSourceECSContainer = "EcsContainer"
)
func resolveCredentials(cfg *aws.Config,
envCfg envConfig, sharedCfg sharedConfig,
handlers request.Handlers,
sessOpts Options,
) (*credentials.Credentials, error) {
// Credentials from Assume Role with specific credentials source.
if envCfg.EnableSharedConfig && len(sharedCfg.AssumeRole.CredentialSource) > 0 {
return resolveCredsFromSource(cfg, envCfg, sharedCfg, handlers, sessOpts)
}
// Credentials from environment variables
if len(envCfg.Creds.AccessKeyID) > 0 {
return credentials.NewStaticCredentialsFromCreds(envCfg.Creds), nil
}
// Fallback to the "default" credential resolution chain.
return resolveCredsFromProfile(cfg, envCfg, sharedCfg, handlers, sessOpts)
}
func resolveCredsFromProfile(cfg *aws.Config,
envCfg envConfig, sharedCfg sharedConfig,
handlers request.Handlers,
sessOpts Options,
) (*credentials.Credentials, error) {
if envCfg.EnableSharedConfig && len(sharedCfg.AssumeRole.RoleARN) > 0 && sharedCfg.AssumeRoleSource != nil {
// Assume IAM role with credentials source from a different profile.
cred, err := resolveCredsFromProfile(cfg, envCfg, *sharedCfg.AssumeRoleSource, handlers, sessOpts)
if err != nil {
return nil, err
}
cfgCp := *cfg
cfgCp.Credentials = cred
return credsFromAssumeRole(cfgCp, handlers, sharedCfg, sessOpts)
} else if len(sharedCfg.Creds.AccessKeyID) > 0 {
// Static Credentials from Shared Config/Credentials file.
return credentials.NewStaticCredentialsFromCreds(
sharedCfg.Creds,
), nil
} else if len(sharedCfg.CredentialProcess) > 0 {
// Credential Process credentials from Shared Config/Credentials file.
return processcreds.NewCredentials(
sharedCfg.CredentialProcess,
), nil
} else if envCfg.EnableSharedConfig && len(sharedCfg.AssumeRole.CredentialSource) > 0 {
// Assume IAM Role with specific credential source.
return resolveCredsFromSource(cfg, envCfg, sharedCfg, handlers, sessOpts)
}
// Fallback to default credentials provider, include mock errors
// for the credential chain so user can identify why credentials
// failed to be retrieved.
return credentials.NewCredentials(&credentials.ChainProvider{
VerboseErrors: aws.BoolValue(cfg.CredentialsChainVerboseErrors),
Providers: []credentials.Provider{
&credProviderError{
Err: awserr.New("EnvAccessKeyNotFound",
"failed to find credentials in the environment.", nil),
},
&credProviderError{
Err: awserr.New("SharedCredsLoad",
fmt.Sprintf("failed to load profile, %s.", envCfg.Profile), nil),
},
defaults.RemoteCredProvider(*cfg, handlers),
},
}), nil
}
func resolveCredsFromSource(cfg *aws.Config,
envCfg envConfig, sharedCfg sharedConfig,
handlers request.Handlers,
sessOpts Options,
) (*credentials.Credentials, error) {
// if both credential_source and source_profile have been set, return an
// error as this is undefined behavior. Only one can be used at a time
// within a profile.
if len(sharedCfg.AssumeRole.SourceProfile) > 0 {
return nil, ErrSharedConfigSourceCollision
}
) (creds *credentials.Credentials, err error) {
cfgCp := *cfg
switch sharedCfg.AssumeRole.CredentialSource {
switch sharedCfg.CredentialSource {
case credSourceEc2Metadata:
p := defaults.RemoteCredProvider(cfgCp, handlers)
cfgCp.Credentials = credentials.NewCredentials(p)
p := defaults.RemoteCredProvider(*cfg, handlers)
creds = credentials.NewCredentials(p)
case credSourceEnvironment:
cfgCp.Credentials = credentials.NewStaticCredentialsFromCreds(envCfg.Creds)
creds = credentials.NewStaticCredentialsFromCreds(envCfg.Creds)
case credSourceECSContainer:
if len(os.Getenv(shareddefaults.ECSCredsProviderEnvVar)) == 0 {
return nil, ErrSharedConfigECSContainerEnvVarEmpty
}
p := defaults.RemoteCredProvider(cfgCp, handlers)
cfgCp.Credentials = credentials.NewCredentials(p)
p := defaults.RemoteCredProvider(*cfg, handlers)
creds = credentials.NewCredentials(p)
default:
return nil, ErrSharedConfigInvalidCredSource
}
return credsFromAssumeRole(cfgCp, handlers, sharedCfg, sessOpts)
return creds, nil
}
func credsFromAssumeRole(cfg aws.Config,
@ -134,7 +191,8 @@ func credsFromAssumeRole(cfg aws.Config,
sharedCfg sharedConfig,
sessOpts Options,
) (*credentials.Credentials, error) {
if len(sharedCfg.AssumeRole.MFASerial) > 0 && sessOpts.AssumeRoleTokenProvider == nil {
if len(sharedCfg.MFASerial) != 0 && sessOpts.AssumeRoleTokenProvider == nil {
// AssumeRole Token provider is required if doing Assume Role
// with MFA.
return nil, AssumeRoleTokenProviderNotSetError{}
@ -145,28 +203,28 @@ func credsFromAssumeRole(cfg aws.Config,
Config: &cfg,
Handlers: handlers.Copy(),
},
sharedCfg.AssumeRole.RoleARN,
sharedCfg.RoleARN,
func(opt *stscreds.AssumeRoleProvider) {
opt.RoleSessionName = sharedCfg.AssumeRole.RoleSessionName
opt.RoleSessionName = sharedCfg.RoleSessionName
opt.Duration = sessOpts.AssumeRoleDuration
// Assume role with external ID
if len(sharedCfg.AssumeRole.ExternalID) > 0 {
opt.ExternalID = aws.String(sharedCfg.AssumeRole.ExternalID)
if len(sharedCfg.ExternalID) > 0 {
opt.ExternalID = aws.String(sharedCfg.ExternalID)
}
// Assume role with MFA
if len(sharedCfg.AssumeRole.MFASerial) > 0 {
opt.SerialNumber = aws.String(sharedCfg.AssumeRole.MFASerial)
if len(sharedCfg.MFASerial) > 0 {
opt.SerialNumber = aws.String(sharedCfg.MFASerial)
opt.TokenProvider = sessOpts.AssumeRoleTokenProvider
}
},
), nil
}
// AssumeRoleTokenProviderNotSetError is an error returned when creating a session when the
// MFAToken option is not set when shared config is configured load assume a
// role with an MFA token.
// AssumeRoleTokenProviderNotSetError is an error returned when creating a
// session when the MFAToken option is not set when shared config is configured
// load assume a role with an MFA token.
type AssumeRoleTokenProviderNotSetError struct{}
// Code is the short id of the error.
@ -193,8 +251,6 @@ type credProviderError struct {
Err error
}
var emptyCreds = credentials.Value{}
func (c credProviderError) Retrieve() (credentials.Value, error) {
return credentials.Value{}, c.Err
}

View file

@ -1,97 +1,93 @@
/*
Package session provides configuration for the SDK's service clients.
Sessions can be shared across all service clients that share the same base
configuration. The Session is built from the SDK's default configuration and
request handlers.
Sessions should be cached when possible, because creating a new Session will
load all configuration values from the environment, and config files each time
the Session is created. Sharing the Session value across all of your service
clients will ensure the configuration is loaded the fewest number of times possible.
Concurrency
Package session provides configuration for the SDK's service clients. Sessions
can be shared across service clients that share the same base configuration.
Sessions are safe to use concurrently as long as the Session is not being
modified. The SDK will not modify the Session once the Session has been created.
Creating service clients concurrently from a shared Session is safe.
modified. Sessions should be cached when possible, because creating a new
Session will load all configuration values from the environment, and config
files each time the Session is created. Sharing the Session value across all of
your service clients will ensure the configuration is loaded the fewest number
of times possible.
Sessions from Shared Config
Sessions can be created using the method above that will only load the
additional config if the AWS_SDK_LOAD_CONFIG environment variable is set.
Alternatively you can explicitly create a Session with shared config enabled.
To do this you can use NewSessionWithOptions to configure how the Session will
be created. Using the NewSessionWithOptions with SharedConfigState set to
SharedConfigEnable will create the session as if the AWS_SDK_LOAD_CONFIG
environment variable was set.
Creating Sessions
When creating Sessions optional aws.Config values can be passed in that will
override the default, or loaded config values the Session is being created
with. This allows you to provide additional, or case based, configuration
as needed.
Sessions options from Shared Config
By default NewSession will only load credentials from the shared credentials
file (~/.aws/credentials). If the AWS_SDK_LOAD_CONFIG environment variable is
set to a truthy value the Session will be created from the configuration
values from the shared config (~/.aws/config) and shared credentials
(~/.aws/credentials) files. See the section Sessions from Shared Config for
more information.
(~/.aws/credentials) files. Using the NewSessionWithOptions with
SharedConfigState set to SharedConfigEnable will create the session as if the
AWS_SDK_LOAD_CONFIG environment variable was set.
Create a Session with the default config and request handlers. With credentials
region, and profile loaded from the environment and shared config automatically.
Requires the AWS_PROFILE to be set, or "default" is used.
Credential and config loading order
The Session will attempt to load configuration and credentials from the
environment, configuration files, and other credential sources. The order
configuration is loaded in is:
* Environment Variables
* Shared Credentials file
* Shared Configuration file (if SharedConfig is enabled)
* EC2 Instance Metadata (credentials only)
The Environment variables for credentials will have precedence over shared
config even if SharedConfig is enabled. To override this behavior, and use
shared config credentials instead specify the session.Options.Profile, (e.g.
when using credential_source=Environment to assume a role).
sess, err := session.NewSessionWithOptions(session.Options{
Profile: "myProfile",
})
Creating Sessions
Creating a Session without additional options will load credentials region, and
profile loaded from the environment and shared config automatically. See,
"Environment Variables" section for information on environment variables used
by Session.
// Create Session
sess := session.Must(session.NewSession())
sess, err := session.NewSession()
When creating Sessions optional aws.Config values can be passed in that will
override the default, or loaded, config values the Session is being created
with. This allows you to provide additional, or case based, configuration
as needed.
// Create a Session with a custom region
sess := session.Must(session.NewSession(&aws.Config{
Region: aws.String("us-east-1"),
}))
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-west-2"),
})
// Create a S3 client instance from a session
sess := session.Must(session.NewSession())
svc := s3.New(sess)
Create Session With Option Overrides
In addition to NewSession, Sessions can be created using NewSessionWithOptions.
This func allows you to control and override how the Session will be created
through code instead of being driven by environment variables only.
Use NewSessionWithOptions when you want to provide the config profile, or
override the shared config state (AWS_SDK_LOAD_CONFIG).
Use NewSessionWithOptions to provide additional configuration driving how the
Session's configuration will be loaded. Such as, specifying shared config
profile, or override the shared config state, (AWS_SDK_LOAD_CONFIG).
// Equivalent to session.NewSession()
sess := session.Must(session.NewSessionWithOptions(session.Options{
sess, err := session.NewSessionWithOptions(session.Options{
// Options
}))
})
// Specify profile to load for the session's config
sess := session.Must(session.NewSessionWithOptions(session.Options{
Profile: "profile_name",
}))
sess, err := session.NewSessionWithOptions(session.Options{
// Specify profile to load for the session's config
Profile: "profile_name",
// Specify profile for config and region for requests
sess := session.Must(session.NewSessionWithOptions(session.Options{
Config: aws.Config{Region: aws.String("us-east-1")},
Profile: "profile_name",
}))
// Provide SDK Config options, such as Region.
Config: aws.Config{
Region: aws.String("us-west-2"),
},
// Force enable Shared Config support
sess := session.Must(session.NewSessionWithOptions(session.Options{
// Force enable Shared Config support
SharedConfigState: session.SharedConfigEnable,
}))
})
Adding Handlers
You can add handlers to a session for processing HTTP requests. All service
clients that use the session inherit the handlers. For example, the following
handler logs every request and its payload made by a service client:
You can add handlers to a session to decorate API operation, (e.g. adding HTTP
headers). All clients that use the Session receive a copy of the Session's
handlers. For example, the following request handler added to the Session logs
every requests made.
// Create a session, and add additional handlers for all service
// clients created with the Session to inherit. Adds logging handler.
@ -99,22 +95,15 @@ handler logs every request and its payload made by a service client:
sess.Handlers.Send.PushFront(func(r *request.Request) {
// Log every request made and its payload
logger.Printf("Request: %s/%s, Payload: %s",
logger.Printf("Request: %s/%s, Params: %s",
r.ClientInfo.ServiceName, r.Operation, r.Params)
})
Deprecated "New" function
The New session function has been deprecated because it does not provide good
way to return errors that occur when loading the configuration files and values.
Because of this, NewSession was created so errors can be retrieved when
creating a session fails.
Shared Config Fields
By default the SDK will only load the shared credentials file's (~/.aws/credentials)
credentials values, and all other config is provided by the environment variables,
SDK defaults, and user provided aws.Config values.
By default the SDK will only load the shared credentials file's
(~/.aws/credentials) credentials values, and all other config is provided by
the environment variables, SDK defaults, and user provided aws.Config values.
If the AWS_SDK_LOAD_CONFIG environment variable is set, or SharedConfigEnable
option is used to create the Session the full shared config values will be
@ -125,24 +114,31 @@ files have the same format.
If both config files are present the configuration from both files will be
read. The Session will be created from configuration values from the shared
credentials file (~/.aws/credentials) over those in the shared config file (~/.aws/config).
credentials file (~/.aws/credentials) over those in the shared config file
(~/.aws/config).
Credentials are the values the SDK should use for authenticating requests with
AWS Services. They are from a configuration file will need to include both
aws_access_key_id and aws_secret_access_key must be provided together in the
same file to be considered valid. The values will be ignored if not a complete
group. aws_session_token is an optional field that can be provided if both of
the other two fields are also provided.
Credentials are the values the SDK uses to authenticating requests with AWS
Services. When specified in a file, both aws_access_key_id and
aws_secret_access_key must be provided together in the same file to be
considered valid. They will be ignored if both are not present.
aws_session_token is an optional field that can be provided in addition to the
other two fields.
aws_access_key_id = AKID
aws_secret_access_key = SECRET
aws_session_token = TOKEN
Assume Role values allow you to configure the SDK to assume an IAM role using
a set of credentials provided in a config file via the source_profile field.
Both "role_arn" and "source_profile" are required. The SDK supports assuming
a role with MFA token if the session option AssumeRoleTokenProvider
is set.
; region only supported if SharedConfigEnabled.
region = us-east-1
Assume Role configuration
The role_arn field allows you to configure the SDK to assume an IAM role using
a set of credentials from another source. Such as when paired with static
credentials, "profile_source", "credential_process", or "credential_source"
fields. If "role_arn" is provided, a source of credentials must also be
specified, such as "source_profile", "credential_source", or
"credential_process".
role_arn = arn:aws:iam::<account_number>:role/<role_name>
source_profile = profile_with_creds
@ -150,40 +146,16 @@ is set.
mfa_serial = <serial or mfa arn>
role_session_name = session_name
Region is the region the SDK should use for looking up AWS service endpoints
and signing requests.
region = us-east-1
Assume Role with MFA token
To create a session with support for assuming an IAM role with MFA set the
session option AssumeRoleTokenProvider to a function that will prompt for the
MFA token code when the SDK assumes the role and refreshes the role's credentials.
This allows you to configure the SDK via the shared config to assumea role
with MFA tokens.
In order for the SDK to assume a role with MFA the SharedConfigState
session option must be set to SharedConfigEnable, or AWS_SDK_LOAD_CONFIG
environment variable set.
The shared configuration instructs the SDK to assume an IAM role with MFA
when the mfa_serial configuration field is set in the shared config
(~/.aws/config) or shared credentials (~/.aws/credentials) file.
If mfa_serial is set in the configuration, the SDK will assume the role, and
the AssumeRoleTokenProvider session option is not set an an error will
be returned when creating the session.
The SDK supports assuming a role with MFA token. If "mfa_serial" is set, you
must also set the Session Option.AssumeRoleTokenProvider. The Session will fail
to load if the AssumeRoleTokenProvider is not specified.
sess := session.Must(session.NewSessionWithOptions(session.Options{
AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
}))
// Create service client value configured for credentials
// from assumed role.
svc := s3.New(sess)
To setup assume role outside of a session see the stscreds.AssumeRoleProvider
To setup Assume Role outside of a session see the stscreds.AssumeRoleProvider
documentation.
Environment Variables

View file

@ -99,21 +99,41 @@ type envConfig struct {
CustomCABundle string
csmEnabled string
CSMEnabled bool
CSMEnabled *bool
CSMPort string
CSMHost string
CSMClientID string
enableEndpointDiscovery string
// Enables endpoint discovery via environment variables.
//
// AWS_ENABLE_ENDPOINT_DISCOVERY=true
EnableEndpointDiscovery *bool
enableEndpointDiscovery string
// Specifies the WebIdentity token the SDK should use to assume a role
// with.
//
// AWS_WEB_IDENTITY_TOKEN_FILE=file_path
WebIdentityTokenFilePath string
// Specifies the IAM role arn to use when assuming an role.
//
// AWS_ROLE_ARN=role_arn
RoleARN string
// Specifies the IAM role session name to use when assuming a role.
//
// AWS_ROLE_SESSION_NAME=session_name
RoleSessionName string
}
var (
csmEnabledEnvKey = []string{
"AWS_CSM_ENABLED",
}
csmHostEnvKey = []string{
"AWS_CSM_HOST",
}
csmPortEnvKey = []string{
"AWS_CSM_PORT",
}
@ -150,6 +170,15 @@ var (
sharedConfigFileEnvKey = []string{
"AWS_CONFIG_FILE",
}
webIdentityTokenFilePathEnvKey = []string{
"AWS_WEB_IDENTITY_TOKEN_FILE",
}
roleARNEnvKey = []string{
"AWS_ROLE_ARN",
}
roleSessionNameEnvKey = []string{
"AWS_ROLE_SESSION_NAME",
}
)
// loadEnvConfig retrieves the SDK's environment configuration.
@ -178,21 +207,33 @@ func envConfigLoad(enableSharedConfig bool) envConfig {
cfg.EnableSharedConfig = enableSharedConfig
setFromEnvVal(&cfg.Creds.AccessKeyID, credAccessEnvKey)
setFromEnvVal(&cfg.Creds.SecretAccessKey, credSecretEnvKey)
setFromEnvVal(&cfg.Creds.SessionToken, credSessionEnvKey)
// Static environment credentials
var creds credentials.Value
setFromEnvVal(&creds.AccessKeyID, credAccessEnvKey)
setFromEnvVal(&creds.SecretAccessKey, credSecretEnvKey)
setFromEnvVal(&creds.SessionToken, credSessionEnvKey)
if creds.HasKeys() {
// Require logical grouping of credentials
creds.ProviderName = EnvProviderName
cfg.Creds = creds
}
// Role Metadata
setFromEnvVal(&cfg.RoleARN, roleARNEnvKey)
setFromEnvVal(&cfg.RoleSessionName, roleSessionNameEnvKey)
// Web identity environment variables
setFromEnvVal(&cfg.WebIdentityTokenFilePath, webIdentityTokenFilePathEnvKey)
// CSM environment variables
setFromEnvVal(&cfg.csmEnabled, csmEnabledEnvKey)
setFromEnvVal(&cfg.CSMHost, csmHostEnvKey)
setFromEnvVal(&cfg.CSMPort, csmPortEnvKey)
setFromEnvVal(&cfg.CSMClientID, csmClientIDEnvKey)
cfg.CSMEnabled = len(cfg.csmEnabled) > 0
// Require logical grouping of credentials
if len(cfg.Creds.AccessKeyID) == 0 || len(cfg.Creds.SecretAccessKey) == 0 {
cfg.Creds = credentials.Value{}
} else {
cfg.Creds.ProviderName = EnvProviderName
if len(cfg.csmEnabled) != 0 {
v, _ := strconv.ParseBool(cfg.csmEnabled)
cfg.CSMEnabled = &v
}
regionKeys := regionEnvKeys

View file

@ -3,6 +3,7 @@ package session
import (
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"io/ioutil"
"net/http"
@ -103,8 +104,20 @@ func New(cfgs ...*aws.Config) *Session {
}
s := deprecatedNewSession(cfgs...)
if envCfg.CSMEnabled {
enableCSM(&s.Handlers, envCfg.CSMClientID, envCfg.CSMPort, s.Config.Logger)
if csmCfg, err := loadCSMConfig(envCfg, []string{}); err != nil {
if l := s.Config.Logger; l != nil {
l.Log(fmt.Sprintf("ERROR: failed to load CSM configuration, %v", err))
}
} else if csmCfg.Enabled {
err := enableCSM(&s.Handlers, csmCfg, s.Config.Logger)
if err != nil {
err = fmt.Errorf("failed to enable CSM, %v", err)
s.Config.Logger.Log("ERROR:", err.Error())
s.Handlers.Validate.PushBack(func(r *request.Request) {
r.Error = err
})
}
}
return s
@ -272,7 +285,7 @@ func NewSessionWithOptions(opts Options) (*Session, error) {
envCfg = loadEnvConfig()
}
if len(opts.Profile) > 0 {
if len(opts.Profile) != 0 {
envCfg.Profile = opts.Profile
}
@ -338,17 +351,18 @@ func deprecatedNewSession(cfgs ...*aws.Config) *Session {
return s
}
func enableCSM(handlers *request.Handlers, clientID string, port string, logger aws.Logger) {
logger.Log("Enabling CSM")
if len(port) == 0 {
port = csm.DefaultPort
func enableCSM(handlers *request.Handlers, cfg csmConfig, logger aws.Logger) error {
if logger != nil {
logger.Log("Enabling CSM")
}
r, err := csm.Start(clientID, "127.0.0.1:"+port)
r, err := csm.Start(cfg.ClientID, csm.AddressWithDefaults(cfg.Host, cfg.Port))
if err != nil {
return
return err
}
r.InjectHandlers(handlers)
return nil
}
func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session, error) {
@ -363,6 +377,7 @@ func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session,
// credentials were.
userCfg := &aws.Config{}
userCfg.MergeIn(cfgs...)
cfg.MergeIn(userCfg)
// Ordered config files will be loaded in with later files overwriting
// previous config file values.
@ -379,9 +394,17 @@ func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session,
}
// Load additional config from file(s)
sharedCfg, err := loadSharedConfig(envCfg.Profile, cfgFiles)
sharedCfg, err := loadSharedConfig(envCfg.Profile, cfgFiles, envCfg.EnableSharedConfig)
if err != nil {
return nil, err
if len(envCfg.Profile) == 0 && !envCfg.EnableSharedConfig && (envCfg.Creds.HasKeys() || userCfg.Credentials != nil) {
// Special case where the user has not explicitly specified an AWS_PROFILE,
// or session.Options.profile, shared config is not enabled, and the
// environment has credentials, allow the shared config file to fail to
// load since the user has already provided credentials, and nothing else
// is required to be read file. Github(aws/aws-sdk-go#2455)
} else if _, ok := err.(SharedConfigProfileNotExistsError); !ok {
return nil, err
}
}
if err := mergeConfigSrcs(cfg, userCfg, envCfg, sharedCfg, handlers, opts); err != nil {
@ -394,8 +417,16 @@ func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session,
}
initHandlers(s)
if envCfg.CSMEnabled {
enableCSM(&s.Handlers, envCfg.CSMClientID, envCfg.CSMPort, s.Config.Logger)
if csmCfg, err := loadCSMConfig(envCfg, cfgFiles); err != nil {
if l := s.Config.Logger; l != nil {
l.Log(fmt.Sprintf("ERROR: failed to load CSM configuration, %v", err))
}
} else if csmCfg.Enabled {
err = enableCSM(&s.Handlers, csmCfg, s.Config.Logger)
if err != nil {
return nil, err
}
}
// Setup HTTP client with custom cert bundle if enabled
@ -408,6 +439,46 @@ func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session,
return s, nil
}
type csmConfig struct {
Enabled bool
Host string
Port string
ClientID string
}
var csmProfileName = "aws_csm"
func loadCSMConfig(envCfg envConfig, cfgFiles []string) (csmConfig, error) {
if envCfg.CSMEnabled != nil {
if *envCfg.CSMEnabled {
return csmConfig{
Enabled: true,
ClientID: envCfg.CSMClientID,
Host: envCfg.CSMHost,
Port: envCfg.CSMPort,
}, nil
}
return csmConfig{}, nil
}
sharedCfg, err := loadSharedConfig(csmProfileName, cfgFiles, false)
if err != nil {
if _, ok := err.(SharedConfigProfileNotExistsError); !ok {
return csmConfig{}, err
}
}
if sharedCfg.CSMEnabled != nil && *sharedCfg.CSMEnabled == true {
return csmConfig{
Enabled: true,
ClientID: sharedCfg.CSMClientID,
Host: sharedCfg.CSMHost,
Port: sharedCfg.CSMPort,
}, nil
}
return csmConfig{}, nil
}
func loadCustomCABundle(s *Session, bundle io.Reader) error {
var t *http.Transport
switch v := s.Config.HTTPClient.Transport.(type) {
@ -461,8 +532,6 @@ func mergeConfigSrcs(cfg, userCfg *aws.Config,
handlers request.Handlers,
sessOpts Options,
) error {
// Merge in user provided configuration
cfg.MergeIn(userCfg)
// Region if not already set by user
if len(aws.StringValue(cfg.Region)) == 0 {

View file

@ -5,7 +5,6 @@ import (
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/internal/ini"
)
@ -23,13 +22,23 @@ const (
mfaSerialKey = `mfa_serial` // optional
roleSessionNameKey = `role_session_name` // optional
// CSM options
csmEnabledKey = `csm_enabled`
csmHostKey = `csm_host`
csmPortKey = `csm_port`
csmClientIDKey = `csm_client_id`
// Additional Config fields
regionKey = `region`
// endpoint discovery group
enableEndpointDiscoveryKey = `endpoint_discovery_enabled` // optional
// External Credential Process
credentialProcessKey = `credential_process`
credentialProcessKey = `credential_process` // optional
// Web Identity Token File
webIdentityTokenFileKey = `web_identity_token_file` // optional
// DefaultSharedConfigProfile is the default profile to be used when
// loading configuration from the config files if another profile name
@ -37,36 +46,33 @@ const (
DefaultSharedConfigProfile = `default`
)
type assumeRoleConfig struct {
RoleARN string
SourceProfile string
CredentialSource string
ExternalID string
MFASerial string
RoleSessionName string
}
// sharedConfig represents the configuration fields of the SDK config files.
type sharedConfig struct {
// Credentials values from the config file. Both aws_access_key_id
// and aws_secret_access_key must be provided together in the same file
// to be considered valid. The values will be ignored if not a complete group.
// aws_session_token is an optional field that can be provided if both of the
// other two fields are also provided.
// Credentials values from the config file. Both aws_access_key_id and
// aws_secret_access_key must be provided together in the same file to be
// considered valid. The values will be ignored if not a complete group.
// aws_session_token is an optional field that can be provided if both of
// the other two fields are also provided.
//
// aws_access_key_id
// aws_secret_access_key
// aws_session_token
Creds credentials.Value
AssumeRole assumeRoleConfig
AssumeRoleSource *sharedConfig
CredentialSource string
CredentialProcess string
WebIdentityTokenFile string
// An external process to request credentials
CredentialProcess string
RoleARN string
RoleSessionName string
ExternalID string
MFASerial string
// Region is the region the SDK should use for looking up AWS service endpoints
// and signing requests.
SourceProfileName string
SourceProfile *sharedConfig
// Region is the region the SDK should use for looking up AWS service
// endpoints and signing requests.
//
// region
Region string
@ -76,6 +82,12 @@ type sharedConfig struct {
//
// endpoint_discovery_enabled = true
EnableEndpointDiscovery *bool
// CSM Options
CSMEnabled *bool
CSMHost string
CSMPort string
CSMClientID string
}
type sharedConfigFile struct {
@ -83,17 +95,18 @@ type sharedConfigFile struct {
IniData ini.Sections
}
// loadSharedConfig retrieves the configuration from the list of files
// using the profile provided. The order the files are listed will determine
// loadSharedConfig retrieves the configuration from the list of files using
// the profile provided. The order the files are listed will determine
// precedence. Values in subsequent files will overwrite values defined in
// earlier files.
//
// For example, given two files A and B. Both define credentials. If the order
// of the files are A then B, B's credential values will be used instead of A's.
// of the files are A then B, B's credential values will be used instead of
// A's.
//
// See sharedConfig.setFromFile for information how the config files
// will be loaded.
func loadSharedConfig(profile string, filenames []string) (sharedConfig, error) {
func loadSharedConfig(profile string, filenames []string, exOpts bool) (sharedConfig, error) {
if len(profile) == 0 {
profile = DefaultSharedConfigProfile
}
@ -104,16 +117,11 @@ func loadSharedConfig(profile string, filenames []string) (sharedConfig, error)
}
cfg := sharedConfig{}
if err = cfg.setFromIniFiles(profile, files); err != nil {
profiles := map[string]struct{}{}
if err = cfg.setFromIniFiles(profiles, profile, files, exOpts); err != nil {
return sharedConfig{}, err
}
if len(cfg.AssumeRole.SourceProfile) > 0 {
if err := cfg.setAssumeRoleSource(profile, files); err != nil {
return sharedConfig{}, err
}
}
return cfg, nil
}
@ -137,70 +145,88 @@ func loadSharedConfigIniFiles(filenames []string) ([]sharedConfigFile, error) {
return files, nil
}
func (cfg *sharedConfig) setAssumeRoleSource(origProfile string, files []sharedConfigFile) error {
var assumeRoleSrc sharedConfig
if len(cfg.AssumeRole.CredentialSource) > 0 {
// setAssumeRoleSource is only called when source_profile is found.
// If both source_profile and credential_source are set, then
// ErrSharedConfigSourceCollision will be returned
return ErrSharedConfigSourceCollision
}
// Multiple level assume role chains are not support
if cfg.AssumeRole.SourceProfile == origProfile {
assumeRoleSrc = *cfg
assumeRoleSrc.AssumeRole = assumeRoleConfig{}
} else {
err := assumeRoleSrc.setFromIniFiles(cfg.AssumeRole.SourceProfile, files)
if err != nil {
return err
}
// Chain if profile depends of other profiles
if len(assumeRoleSrc.AssumeRole.SourceProfile) > 0 {
err := assumeRoleSrc.setAssumeRoleSource(cfg.AssumeRole.SourceProfile, files)
if err != nil {
return err
}
}
}
if cfg.AssumeRole.SourceProfile == origProfile || len(assumeRoleSrc.AssumeRole.SourceProfile) == 0 {
if len(assumeRoleSrc.AssumeRole.CredentialSource) == 0 && len(assumeRoleSrc.Creds.AccessKeyID) == 0 {
return SharedConfigAssumeRoleError{RoleARN: cfg.AssumeRole.RoleARN}
}
}
cfg.AssumeRoleSource = &assumeRoleSrc
return nil
}
func (cfg *sharedConfig) setFromIniFiles(profile string, files []sharedConfigFile) error {
func (cfg *sharedConfig) setFromIniFiles(profiles map[string]struct{}, profile string, files []sharedConfigFile, exOpts bool) error {
// Trim files from the list that don't exist.
var skippedFiles int
var profileNotFoundErr error
for _, f := range files {
if err := cfg.setFromIniFile(profile, f); err != nil {
if err := cfg.setFromIniFile(profile, f, exOpts); err != nil {
if _, ok := err.(SharedConfigProfileNotExistsError); ok {
// Ignore proviles missings
// Ignore profiles not defined in individual files.
profileNotFoundErr = err
skippedFiles++
continue
}
return err
}
}
if skippedFiles == len(files) {
// If all files were skipped because the profile is not found, return
// the original profile not found error.
return profileNotFoundErr
}
if _, ok := profiles[profile]; ok {
// if this is the second instance of the profile the Assume Role
// options must be cleared because they are only valid for the
// first reference of a profile. The self linked instance of the
// profile only have credential provider options.
cfg.clearAssumeRoleOptions()
} else {
// First time a profile has been seen, It must either be a assume role
// or credentials. Assert if the credential type requires a role ARN,
// the ARN is also set.
if err := cfg.validateCredentialsRequireARN(profile); err != nil {
return err
}
}
profiles[profile] = struct{}{}
if err := cfg.validateCredentialType(); err != nil {
return err
}
// Link source profiles for assume roles
if len(cfg.SourceProfileName) != 0 {
// Linked profile via source_profile ignore credential provider
// options, the source profile must provide the credentials.
cfg.clearCredentialOptions()
srcCfg := &sharedConfig{}
err := srcCfg.setFromIniFiles(profiles, cfg.SourceProfileName, files, exOpts)
if err != nil {
// SourceProfile that doesn't exist is an error in configuration.
if _, ok := err.(SharedConfigProfileNotExistsError); ok {
err = SharedConfigAssumeRoleError{
RoleARN: cfg.RoleARN,
SourceProfile: cfg.SourceProfileName,
}
}
return err
}
if !srcCfg.hasCredentials() {
return SharedConfigAssumeRoleError{
RoleARN: cfg.RoleARN,
SourceProfile: cfg.SourceProfileName,
}
}
cfg.SourceProfile = srcCfg
}
return nil
}
// setFromFile loads the configuration from the file using
// the profile provided. A sharedConfig pointer type value is used so that
// multiple config file loadings can be chained.
// setFromFile loads the configuration from the file using the profile
// provided. A sharedConfig pointer type value is used so that multiple config
// file loadings can be chained.
//
// Only loads complete logically grouped values, and will not set fields in cfg
// for incomplete grouped values in the config. Such as credentials. For example
// if a config file only includes aws_access_key_id but no aws_secret_access_key
// the aws_access_key_id will be ignored.
func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile) error {
// for incomplete grouped values in the config. Such as credentials. For
// example if a config file only includes aws_access_key_id but no
// aws_secret_access_key the aws_access_key_id will be ignored.
func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, exOpts bool) error {
section, ok := file.IniData.GetSection(profile)
if !ok {
// Fallback to to alternate profile name: profile <name>
@ -210,53 +236,143 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile) e
}
}
if exOpts {
// Assume Role Parameters
updateString(&cfg.RoleARN, section, roleArnKey)
updateString(&cfg.ExternalID, section, externalIDKey)
updateString(&cfg.MFASerial, section, mfaSerialKey)
updateString(&cfg.RoleSessionName, section, roleSessionNameKey)
updateString(&cfg.SourceProfileName, section, sourceProfileKey)
updateString(&cfg.CredentialSource, section, credentialSourceKey)
updateString(&cfg.Region, section, regionKey)
}
updateString(&cfg.CredentialProcess, section, credentialProcessKey)
updateString(&cfg.WebIdentityTokenFile, section, webIdentityTokenFileKey)
// Shared Credentials
akid := section.String(accessKeyIDKey)
secret := section.String(secretAccessKey)
if len(akid) > 0 && len(secret) > 0 {
cfg.Creds = credentials.Value{
AccessKeyID: akid,
SecretAccessKey: secret,
SessionToken: section.String(sessionTokenKey),
ProviderName: fmt.Sprintf("SharedConfigCredentials: %s", file.Filename),
}
creds := credentials.Value{
AccessKeyID: section.String(accessKeyIDKey),
SecretAccessKey: section.String(secretAccessKey),
SessionToken: section.String(sessionTokenKey),
ProviderName: fmt.Sprintf("SharedConfigCredentials: %s", file.Filename),
}
// Assume Role
roleArn := section.String(roleArnKey)
srcProfile := section.String(sourceProfileKey)
credentialSource := section.String(credentialSourceKey)
hasSource := len(srcProfile) > 0 || len(credentialSource) > 0
if len(roleArn) > 0 && hasSource {
cfg.AssumeRole = assumeRoleConfig{
RoleARN: roleArn,
SourceProfile: srcProfile,
CredentialSource: credentialSource,
ExternalID: section.String(externalIDKey),
MFASerial: section.String(mfaSerialKey),
RoleSessionName: section.String(roleSessionNameKey),
}
}
// `credential_process`
if credProc := section.String(credentialProcessKey); len(credProc) > 0 {
cfg.CredentialProcess = credProc
}
// Region
if v := section.String(regionKey); len(v) > 0 {
cfg.Region = v
if creds.HasKeys() {
cfg.Creds = creds
}
// Endpoint discovery
if section.Has(enableEndpointDiscoveryKey) {
v := section.Bool(enableEndpointDiscoveryKey)
cfg.EnableEndpointDiscovery = &v
updateBoolPtr(&cfg.EnableEndpointDiscovery, section, enableEndpointDiscoveryKey)
// CSM options
updateBoolPtr(&cfg.CSMEnabled, section, csmEnabledKey)
updateString(&cfg.CSMHost, section, csmHostKey)
updateString(&cfg.CSMPort, section, csmPortKey)
updateString(&cfg.CSMClientID, section, csmClientIDKey)
return nil
}
func (cfg *sharedConfig) validateCredentialsRequireARN(profile string) error {
var credSource string
switch {
case len(cfg.SourceProfileName) != 0:
credSource = sourceProfileKey
case len(cfg.CredentialSource) != 0:
credSource = credentialSourceKey
case len(cfg.WebIdentityTokenFile) != 0:
credSource = webIdentityTokenFileKey
}
if len(credSource) != 0 && len(cfg.RoleARN) == 0 {
return CredentialRequiresARNError{
Type: credSource,
Profile: profile,
}
}
return nil
}
func (cfg *sharedConfig) validateCredentialType() error {
// Only one or no credential type can be defined.
if !oneOrNone(
len(cfg.SourceProfileName) != 0,
len(cfg.CredentialSource) != 0,
len(cfg.CredentialProcess) != 0,
len(cfg.WebIdentityTokenFile) != 0,
) {
return ErrSharedConfigSourceCollision
}
return nil
}
func (cfg *sharedConfig) hasCredentials() bool {
switch {
case len(cfg.SourceProfileName) != 0:
case len(cfg.CredentialSource) != 0:
case len(cfg.CredentialProcess) != 0:
case len(cfg.WebIdentityTokenFile) != 0:
case cfg.Creds.HasKeys():
default:
return false
}
return true
}
func (cfg *sharedConfig) clearCredentialOptions() {
cfg.CredentialSource = ""
cfg.CredentialProcess = ""
cfg.WebIdentityTokenFile = ""
cfg.Creds = credentials.Value{}
}
func (cfg *sharedConfig) clearAssumeRoleOptions() {
cfg.RoleARN = ""
cfg.ExternalID = ""
cfg.MFASerial = ""
cfg.RoleSessionName = ""
cfg.SourceProfileName = ""
}
func oneOrNone(bs ...bool) bool {
var count int
for _, b := range bs {
if b {
count++
if count > 1 {
return false
}
}
}
return true
}
// updateString will only update the dst with the value in the section key, key
// is present in the section.
func updateString(dst *string, section ini.Section, key string) {
if !section.Has(key) {
return
}
*dst = section.String(key)
}
// updateBoolPtr will only update the dst with the value in the section key,
// key is present in the section.
func updateBoolPtr(dst **bool, section ini.Section, key string) {
if !section.Has(key) {
return
}
*dst = new(bool)
**dst = section.Bool(key)
}
// SharedConfigLoadError is an error for the shared config file failed to load.
type SharedConfigLoadError struct {
Filename string
@ -314,7 +430,8 @@ func (e SharedConfigProfileNotExistsError) Error() string {
// profile contains assume role information, but that information is invalid
// or not complete.
type SharedConfigAssumeRoleError struct {
RoleARN string
RoleARN string
SourceProfile string
}
// Code is the short id of the error.
@ -324,8 +441,10 @@ func (e SharedConfigAssumeRoleError) Code() string {
// Message is the description of the error
func (e SharedConfigAssumeRoleError) Message() string {
return fmt.Sprintf("failed to load assume role for %s, source profile has no shared credentials",
e.RoleARN)
return fmt.Sprintf(
"failed to load assume role for %s, source profile %s has no shared credentials",
e.RoleARN, e.SourceProfile,
)
}
// OrigErr is the underlying error that caused the failure.
@ -337,3 +456,36 @@ func (e SharedConfigAssumeRoleError) OrigErr() error {
func (e SharedConfigAssumeRoleError) Error() string {
return awserr.SprintError(e.Code(), e.Message(), "", nil)
}
// CredentialRequiresARNError provides the error for shared config credentials
// that are incorrectly configured in the shared config or credentials file.
type CredentialRequiresARNError struct {
// type of credentials that were configured.
Type string
// Profile name the credentials were in.
Profile string
}
// Code is the short id of the error.
func (e CredentialRequiresARNError) Code() string {
return "CredentialRequiresARNError"
}
// Message is the description of the error
func (e CredentialRequiresARNError) Message() string {
return fmt.Sprintf(
"credential type %s requires role_arn, profile %s",
e.Type, e.Profile,
)
}
// OrigErr is the underlying error that caused the failure.
func (e CredentialRequiresARNError) OrigErr() error {
return nil
}
// Error satisfies the error interface.
func (e CredentialRequiresARNError) Error() string {
return awserr.SprintError(e.Code(), e.Message(), "", nil)
}

View file

@ -687,7 +687,11 @@ func (ctx *signingCtx) buildBodyDigest() error {
if !aws.IsReaderSeekable(ctx.Body) {
return fmt.Errorf("cannot use unseekable request body %T, for signed request with body", ctx.Body)
}
hash = hex.EncodeToString(makeSha256Reader(ctx.Body))
hashBytes, err := makeSha256Reader(ctx.Body)
if err != nil {
return err
}
hash = hex.EncodeToString(hashBytes)
}
if includeSHA256Header {
@ -734,10 +738,16 @@ func makeSha256(data []byte) []byte {
return hash.Sum(nil)
}
func makeSha256Reader(reader io.ReadSeeker) []byte {
func makeSha256Reader(reader io.ReadSeeker) (hashBytes []byte, err error) {
hash := sha256.New()
start, _ := reader.Seek(0, sdkio.SeekCurrent)
defer reader.Seek(start, sdkio.SeekStart)
start, err := reader.Seek(0, sdkio.SeekCurrent)
if err != nil {
return nil, err
}
defer func() {
// ensure error is return if unable to seek back to start of payload.
_, err = reader.Seek(start, sdkio.SeekStart)
}()
// Use CopyN to avoid allocating the 32KB buffer in io.Copy for bodies
// smaller than 32KB. Fall back to io.Copy if we fail to determine the size.
@ -748,7 +758,7 @@ func makeSha256Reader(reader io.ReadSeeker) []byte {
io.CopyN(hash, reader, size)
}
return hash.Sum(nil)
return hash.Sum(nil), nil
}
const doubleSpace = " "

View file

@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.20.10"
const SDKVersion = "1.23.8"

View file

@ -146,6 +146,9 @@ func unmarshalStatusCode(v reflect.Value, statusCode int) {
}
func unmarshalHeaderMap(r reflect.Value, headers http.Header, prefix string) error {
if len(headers) == 0 {
return nil
}
switch r.Interface().(type) {
case map[string]*string: // we only support string map value types
out := map[string]*string{}
@ -155,19 +158,28 @@ func unmarshalHeaderMap(r reflect.Value, headers http.Header, prefix string) err
out[k[len(prefix):]] = &v[0]
}
}
r.Set(reflect.ValueOf(out))
if len(out) != 0 {
r.Set(reflect.ValueOf(out))
}
}
return nil
}
func unmarshalHeader(v reflect.Value, header string, tag reflect.StructTag) error {
isJSONValue := tag.Get("type") == "jsonvalue"
if isJSONValue {
switch tag.Get("type") {
case "jsonvalue":
if len(header) == 0 {
return nil
}
} else if !v.IsValid() || (header == "" && v.Elem().Kind() != reflect.String) {
return nil
case "blob":
if len(header) == 0 {
return nil
}
default:
if !v.IsValid() || (header == "" && v.Elem().Kind() != reflect.String) {
return nil
}
}
switch v.Interface().(type) {
@ -178,7 +190,7 @@ func unmarshalHeader(v reflect.Value, header string, tag reflect.StructTag) erro
if err != nil {
return err
}
v.Set(reflect.ValueOf(&b))
v.Set(reflect.ValueOf(b))
case *bool:
b, err := strconv.ParseBool(header)
if err != nil {

View file

@ -39,7 +39,7 @@ func Build(r *request.Request) {
r.Error = awserr.NewRequestFailure(
awserr.New(request.ErrCodeSerialization,
"failed to encode rest XML request", err),
r.HTTPResponse.StatusCode,
0,
r.RequestID,
)
return

View file

@ -8516,6 +8516,11 @@ type CopyObjectInput struct {
// key was transmitted without error.
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
// Specifies the AWS KMS Encryption Context to use for object encryption. The
// value of this header is a base64-encoded UTF-8 string holding JSON with the
// encryption context key-value pairs.
SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"`
// Specifies the AWS KMS key ID to use for object encryption. All GET and PUT
// requests for an object protected by AWS KMS will fail if not made via SSL
// or using SigV4. Documentation on configuring any of the officially supported
@ -8780,6 +8785,12 @@ func (s *CopyObjectInput) SetSSECustomerKeyMD5(v string) *CopyObjectInput {
return s
}
// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value.
func (s *CopyObjectInput) SetSSEKMSEncryptionContext(v string) *CopyObjectInput {
s.SSEKMSEncryptionContext = &v
return s
}
// SetSSEKMSKeyId sets the SSEKMSKeyId field's value.
func (s *CopyObjectInput) SetSSEKMSKeyId(v string) *CopyObjectInput {
s.SSEKMSKeyId = &v
@ -8840,6 +8851,11 @@ type CopyObjectOutput struct {
// verification of the customer-provided encryption key.
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
// If present, specifies the AWS KMS Encryption Context to use for object encryption.
// The value of this header is a base64-encoded UTF-8 string holding JSON with
// the encryption context key-value pairs.
SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"`
// If present, specifies the ID of the AWS Key Management Service (KMS) master
// encryption key that was used for the object.
SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"`
@ -8898,6 +8914,12 @@ func (s *CopyObjectOutput) SetSSECustomerKeyMD5(v string) *CopyObjectOutput {
return s
}
// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value.
func (s *CopyObjectOutput) SetSSEKMSEncryptionContext(v string) *CopyObjectOutput {
s.SSEKMSEncryptionContext = &v
return s
}
// SetSSEKMSKeyId sets the SSEKMSKeyId field's value.
func (s *CopyObjectOutput) SetSSEKMSKeyId(v string) *CopyObjectOutput {
s.SSEKMSKeyId = &v
@ -9220,6 +9242,11 @@ type CreateMultipartUploadInput struct {
// key was transmitted without error.
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
// Specifies the AWS KMS Encryption Context to use for object encryption. The
// value of this header is a base64-encoded UTF-8 string holding JSON with the
// encryption context key-value pairs.
SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"`
// Specifies the AWS KMS key ID to use for object encryption. All GET and PUT
// requests for an object protected by AWS KMS will fail if not made via SSL
// or using SigV4. Documentation on configuring any of the officially supported
@ -9414,6 +9441,12 @@ func (s *CreateMultipartUploadInput) SetSSECustomerKeyMD5(v string) *CreateMulti
return s
}
// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value.
func (s *CreateMultipartUploadInput) SetSSEKMSEncryptionContext(v string) *CreateMultipartUploadInput {
s.SSEKMSEncryptionContext = &v
return s
}
// SetSSEKMSKeyId sets the SSEKMSKeyId field's value.
func (s *CreateMultipartUploadInput) SetSSEKMSKeyId(v string) *CreateMultipartUploadInput {
s.SSEKMSKeyId = &v
@ -9474,6 +9507,11 @@ type CreateMultipartUploadOutput struct {
// verification of the customer-provided encryption key.
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
// If present, specifies the AWS KMS Encryption Context to use for object encryption.
// The value of this header is a base64-encoded UTF-8 string holding JSON with
// the encryption context key-value pairs.
SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"`
// If present, specifies the ID of the AWS Key Management Service (KMS) master
// encryption key that was used for the object.
SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"`
@ -9545,6 +9583,12 @@ func (s *CreateMultipartUploadOutput) SetSSECustomerKeyMD5(v string) *CreateMult
return s
}
// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value.
func (s *CreateMultipartUploadOutput) SetSSEKMSEncryptionContext(v string) *CreateMultipartUploadOutput {
s.SSEKMSEncryptionContext = &v
return s
}
// SetSSEKMSKeyId sets the SSEKMSKeyId field's value.
func (s *CreateMultipartUploadOutput) SetSSEKMSKeyId(v string) *CreateMultipartUploadOutput {
s.SSEKMSKeyId = &v
@ -20372,6 +20416,11 @@ type PutObjectInput struct {
// key was transmitted without error.
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
// Specifies the AWS KMS Encryption Context to use for object encryption. The
// value of this header is a base64-encoded UTF-8 string holding JSON with the
// encryption context key-value pairs.
SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"`
// Specifies the AWS KMS key ID to use for object encryption. All GET and PUT
// requests for an object protected by AWS KMS will fail if not made via SSL
// or using SigV4. Documentation on configuring any of the officially supported
@ -20585,6 +20634,12 @@ func (s *PutObjectInput) SetSSECustomerKeyMD5(v string) *PutObjectInput {
return s
}
// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value.
func (s *PutObjectInput) SetSSEKMSEncryptionContext(v string) *PutObjectInput {
s.SSEKMSEncryptionContext = &v
return s
}
// SetSSEKMSKeyId sets the SSEKMSKeyId field's value.
func (s *PutObjectInput) SetSSEKMSKeyId(v string) *PutObjectInput {
s.SSEKMSKeyId = &v
@ -20861,6 +20916,11 @@ type PutObjectOutput struct {
// verification of the customer-provided encryption key.
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
// If present, specifies the AWS KMS Encryption Context to use for object encryption.
// The value of this header is a base64-encoded UTF-8 string holding JSON with
// the encryption context key-value pairs.
SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"`
// If present, specifies the ID of the AWS Key Management Service (KMS) master
// encryption key that was used for the object.
SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"`
@ -20913,6 +20973,12 @@ func (s *PutObjectOutput) SetSSECustomerKeyMD5(v string) *PutObjectOutput {
return s
}
// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value.
func (s *PutObjectOutput) SetSSEKMSEncryptionContext(v string) *PutObjectOutput {
s.SSEKMSEncryptionContext = &v
return s
}
// SetSSEKMSKeyId sets the SSEKMSKeyId field's value.
func (s *PutObjectOutput) SetSSEKMSKeyId(v string) *PutObjectOutput {
s.SSEKMSKeyId = &v

View file

@ -63,6 +63,20 @@
// See the s3manager package's Downloader type documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#Downloader
//
// Automatic URI cleaning
//
// Interacting with objects whose keys contain adjacent slashes (e.g. bucketname/foo//bar/objectname)
// requires setting DisableRestProtocolURICleaning to true in the aws.Config struct
// used by the service client.
//
// svc := s3.New(sess, &aws.Config{
// DisableRestProtocolURICleaning: aws.Bool(true),
// })
// out, err := svc.GetObject(&s3.GetObjectInput {
// Bucket: aws.String("bucketname"),
// Key: aws.String("//foo//bar//moo"),
// })
//
// Get Bucket Region
//
// GetBucketRegion will attempt to get the region for a bucket using a region

View file

@ -99,7 +99,7 @@ func NewDownloader(c client.ConfigProvider, options ...func(*Downloader)) *Downl
// sess := session.Must(session.NewSession())
//
// // The S3 client the S3 Downloader will use
// s3Svc := s3.new(sess)
// s3Svc := s3.New(sess)
//
// // Create a downloader with the s3 client and default options
// downloader := s3manager.NewDownloaderWithClient(s3Svc)
@ -126,7 +126,8 @@ type maxRetrier interface {
}
// Download downloads an object in S3 and writes the payload into w using
// concurrent GET requests.
// concurrent GET requests. The n int64 returned is the size of the object downloaded
// in bytes.
//
// Additional functional options can be provided to configure the individual
// download. These options are copies of the Downloader instance Download is called from.
@ -148,7 +149,8 @@ func (d Downloader) Download(w io.WriterAt, input *s3.GetObjectInput, options ..
}
// DownloadWithContext downloads an object in S3 and writes the payload into w
// using concurrent GET requests.
// using concurrent GET requests. The n int64 returned is the size of the object downloaded
// in bytes.
//
// DownloadWithContext is the same as Download with the additional support for
// Context input parameters. The Context must not be nil. A nil Context will

View file

@ -363,7 +363,9 @@ type uploader struct {
// internal logic for deciding whether to upload a single part or use a
// multipart upload.
func (u *uploader) upload() (*UploadOutput, error) {
u.init()
if err := u.init(); err != nil {
return nil, awserr.New("ReadRequestBody", "unable to initialize upload", err)
}
if u.cfg.PartSize < MinUploadPartSize {
msg := fmt.Sprintf("part size must be at least %d bytes", MinUploadPartSize)
@ -383,7 +385,7 @@ func (u *uploader) upload() (*UploadOutput, error) {
}
// init will initialize all default options.
func (u *uploader) init() {
func (u *uploader) init() error {
if u.cfg.Concurrency == 0 {
u.cfg.Concurrency = DefaultUploadConcurrency
}
@ -399,19 +401,19 @@ func (u *uploader) init() {
}
// Try to get the total size for some optimizations
u.initSize()
return u.initSize()
}
// initSize tries to detect the total stream size, setting u.totalSize. If
// the size is not known, totalSize is set to -1.
func (u *uploader) initSize() {
func (u *uploader) initSize() error {
u.totalSize = -1
switch r := u.in.Body.(type) {
case io.Seeker:
n, err := aws.SeekerLen(r)
if err != nil {
return
return err
}
u.totalSize = n
@ -423,6 +425,8 @@ func (u *uploader) initSize() {
u.cfg.PartSize = (u.totalSize / int64(u.cfg.MaxUploadParts)) + 1
}
}
return nil
}
// nextReader returns a seekable reader representing the next packet of data.

View file

@ -100,6 +100,11 @@ type UploadInput struct {
// key was transmitted without error.
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
// Specifies the AWS KMS Encryption Context to use for object encryption. The
// value of this header is a base64-encoded UTF-8 string holding JSON with the
// encryption context key-value pairs.
SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"`
// Specifies the AWS KMS key ID to use for object encryption. All GET and PUT
// requests for an object protected by AWS KMS will fail if not made via SSL
// or using SigV4. Documentation on configuring any of the officially supported

View file

@ -702,6 +702,102 @@ func (c *STS) DecodeAuthorizationMessageWithContext(ctx aws.Context, input *Deco
return out, req.Send()
}
const opGetAccessKeyInfo = "GetAccessKeyInfo"
// GetAccessKeyInfoRequest generates a "aws/request.Request" representing the
// client's request for the GetAccessKeyInfo operation. The "output" return
// value will be populated with the request's response once the request completes
// successfully.
//
// Use "Send" method on the returned Request to send the API call to the service.
// the "output" return value is not valid until after Send returns without error.
//
// See GetAccessKeyInfo for more information on using the GetAccessKeyInfo
// API call, and error handling.
//
// This method is useful when you want to inject custom logic or configuration
// into the SDK's request lifecycle. Such as custom headers, or retry logic.
//
//
// // Example sending a request using the GetAccessKeyInfoRequest method.
// req, resp := client.GetAccessKeyInfoRequest(params)
//
// err := req.Send()
// if err == nil { // resp is now filled
// fmt.Println(resp)
// }
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/GetAccessKeyInfo
func (c *STS) GetAccessKeyInfoRequest(input *GetAccessKeyInfoInput) (req *request.Request, output *GetAccessKeyInfoOutput) {
op := &request.Operation{
Name: opGetAccessKeyInfo,
HTTPMethod: "POST",
HTTPPath: "/",
}
if input == nil {
input = &GetAccessKeyInfoInput{}
}
output = &GetAccessKeyInfoOutput{}
req = c.newRequest(op, input, output)
return
}
// GetAccessKeyInfo API operation for AWS Security Token Service.
//
// Returns the account identifier for the specified access key ID.
//
// Access keys consist of two parts: an access key ID (for example, AKIAIOSFODNN7EXAMPLE)
// and a secret access key (for example, wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY).
// For more information about access keys, see Managing Access Keys for IAM
// Users (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html)
// in the IAM User Guide.
//
// When you pass an access key ID to this operation, it returns the ID of the
// AWS account to which the keys belong. Access key IDs beginning with AKIA
// are long-term credentials for an IAM user or the AWS account root user. Access
// key IDs beginning with ASIA are temporary credentials that are created using
// STS operations. If the account in the response belongs to you, you can sign
// in as the root user and review your root user access keys. Then, you can
// pull a credentials report (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_getting-report.html)
// to learn which IAM user owns the keys. To learn who requested the temporary
// credentials for an ASIA access key, view the STS events in your CloudTrail
// logs (https://docs.aws.amazon.com/IAM/latest/UserGuide/cloudtrail-integration.html).
//
// This operation does not indicate the state of the access key. The key might
// be active, inactive, or deleted. Active keys might not have permissions to
// perform an operation. Providing a deleted access key might return an error
// that the key doesn't exist.
//
// Returns awserr.Error for service API and SDK errors. Use runtime type assertions
// with awserr.Error's Code and Message methods to get detailed information about
// the error.
//
// See the AWS API reference guide for AWS Security Token Service's
// API operation GetAccessKeyInfo for usage and error information.
// See also, https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/GetAccessKeyInfo
func (c *STS) GetAccessKeyInfo(input *GetAccessKeyInfoInput) (*GetAccessKeyInfoOutput, error) {
req, out := c.GetAccessKeyInfoRequest(input)
return out, req.Send()
}
// GetAccessKeyInfoWithContext is the same as GetAccessKeyInfo with the addition of
// the ability to pass a context and additional request options.
//
// See GetAccessKeyInfo for details on how to use this API operation.
//
// The context must be non-nil and will be used for request cancellation. If
// the context is nil a panic will occur. In the future the SDK may create
// sub-contexts for http.Requests. See https://golang.org/pkg/context/
// for more information on using Contexts.
func (c *STS) GetAccessKeyInfoWithContext(ctx aws.Context, input *GetAccessKeyInfoInput, opts ...request.Option) (*GetAccessKeyInfoOutput, error) {
req, out := c.GetAccessKeyInfoRequest(input)
req.SetContext(ctx)
req.ApplyOptions(opts...)
return out, req.Send()
}
const opGetCallerIdentity = "GetCallerIdentity"
// GetCallerIdentityRequest generates a "aws/request.Request" representing the
@ -746,8 +842,15 @@ func (c *STS) GetCallerIdentityRequest(input *GetCallerIdentityInput) (req *requ
// GetCallerIdentity API operation for AWS Security Token Service.
//
// Returns details about the IAM identity whose credentials are used to call
// the API.
// Returns details about the IAM user or role whose credentials are used to
// call the operation.
//
// No permissions are required to perform this operation. If an administrator
// adds a policy to your IAM user or role that explicitly denies access to the
// sts:GetCallerIdentity action, you can still perform this operation. Permissions
// are not required because the same information is returned when an IAM user
// or role is denied access. To view an example response, see I Am Not Authorized
// to Perform: iam:DeleteVirtualMFADevice (https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_general.html#troubleshoot_general_access-denied-delete-mfa).
//
// Returns awserr.Error for service API and SDK errors. Use runtime type assertions
// with awserr.Error's Code and Message methods to get detailed information about
@ -1129,7 +1232,7 @@ type AssumeRoleInput struct {
// This parameter is optional. You can provide up to 10 managed policy ARNs.
// However, the plain text that you use for both inline and managed session
// policies shouldn't exceed 2048 characters. For more information about ARNs,
// see Amazon Resource Names (ARNs) and AWS Service Namespaces (general/latest/gr/aws-arns-and-namespaces.html)
// see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// in the AWS General Reference.
//
// The characters in this parameter count towards the 2048 character session
@ -1407,7 +1510,7 @@ type AssumeRoleWithSAMLInput struct {
// This parameter is optional. You can provide up to 10 managed policy ARNs.
// However, the plain text that you use for both inline and managed session
// policies shouldn't exceed 2048 characters. For more information about ARNs,
// see Amazon Resource Names (ARNs) and AWS Service Namespaces (general/latest/gr/aws-arns-and-namespaces.html)
// see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// in the AWS General Reference.
//
// The characters in this parameter count towards the 2048 character session
@ -1702,7 +1805,7 @@ type AssumeRoleWithWebIdentityInput struct {
// This parameter is optional. You can provide up to 10 managed policy ARNs.
// However, the plain text that you use for both inline and managed session
// policies shouldn't exceed 2048 characters. For more information about ARNs,
// see Amazon Resource Names (ARNs) and AWS Service Namespaces (general/latest/gr/aws-arns-and-namespaces.html)
// see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// in the AWS General Reference.
//
// The characters in this parameter count towards the 2048 character session
@ -2156,6 +2259,73 @@ func (s *FederatedUser) SetFederatedUserId(v string) *FederatedUser {
return s
}
type GetAccessKeyInfoInput struct {
_ struct{} `type:"structure"`
// The identifier of an access key.
//
// This parameter allows (through its regex pattern) a string of characters
// that can consist of any upper- or lowercased letter or digit.
//
// AccessKeyId is a required field
AccessKeyId *string `min:"16" type:"string" required:"true"`
}
// String returns the string representation
func (s GetAccessKeyInfoInput) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s GetAccessKeyInfoInput) GoString() string {
return s.String()
}
// Validate inspects the fields of the type to determine if they are valid.
func (s *GetAccessKeyInfoInput) Validate() error {
invalidParams := request.ErrInvalidParams{Context: "GetAccessKeyInfoInput"}
if s.AccessKeyId == nil {
invalidParams.Add(request.NewErrParamRequired("AccessKeyId"))
}
if s.AccessKeyId != nil && len(*s.AccessKeyId) < 16 {
invalidParams.Add(request.NewErrParamMinLen("AccessKeyId", 16))
}
if invalidParams.Len() > 0 {
return invalidParams
}
return nil
}
// SetAccessKeyId sets the AccessKeyId field's value.
func (s *GetAccessKeyInfoInput) SetAccessKeyId(v string) *GetAccessKeyInfoInput {
s.AccessKeyId = &v
return s
}
type GetAccessKeyInfoOutput struct {
_ struct{} `type:"structure"`
// The number used to identify the AWS account.
Account *string `type:"string"`
}
// String returns the string representation
func (s GetAccessKeyInfoOutput) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s GetAccessKeyInfoOutput) GoString() string {
return s.String()
}
// SetAccount sets the Account field's value.
func (s *GetAccessKeyInfoOutput) SetAccount(v string) *GetAccessKeyInfoOutput {
s.Account = &v
return s
}
type GetCallerIdentityInput struct {
_ struct{} `type:"structure"`
}
@ -2284,7 +2454,7 @@ type GetFederationTokenInput struct {
// use as managed session policies. The plain text that you use for both inline
// and managed session policies shouldn't exceed 2048 characters. You can provide
// up to 10 managed policy ARNs. For more information about ARNs, see Amazon
// Resource Names (ARNs) and AWS Service Namespaces (general/latest/gr/aws-arns-and-namespaces.html)
// Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// in the AWS General Reference.
//
// This parameter is optional. However, if you do not pass any session policies,
@ -2545,7 +2715,7 @@ type PolicyDescriptorType struct {
// The Amazon Resource Name (ARN) of the IAM managed policy to use as a session
// policy for the role. For more information about ARNs, see Amazon Resource
// Names (ARNs) and AWS Service Namespaces (general/latest/gr/aws-arns-and-namespaces.html)
// Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
// in the AWS General Reference.
Arn *string `locationName:"arn" min:"20" type:"string"`
}

View file

@ -0,0 +1,11 @@
package sts
import "github.com/aws/aws-sdk-go/aws/request"
func init() {
initRequest = customizeRequest
}
func customizeRequest(r *request.Request) {
r.RetryErrorCodes = append(r.RetryErrorCodes, ErrCodeIDPCommunicationErrorException)
}

View file

@ -0,0 +1,96 @@
// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
// Package stsiface provides an interface to enable mocking the AWS Security Token Service service client
// for testing your code.
//
// It is important to note that this interface will have breaking changes
// when the service model is updated and adds new API operations, paginators,
// and waiters.
package stsiface
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/sts"
)
// STSAPI provides an interface to enable mocking the
// sts.STS service client's API operation,
// paginators, and waiters. This make unit testing your code that calls out
// to the SDK's service client's calls easier.
//
// The best way to use this interface is so the SDK's service client's calls
// can be stubbed out for unit testing your code with the SDK without needing
// to inject custom request handlers into the SDK's request pipeline.
//
// // myFunc uses an SDK service client to make a request to
// // AWS Security Token Service.
// func myFunc(svc stsiface.STSAPI) bool {
// // Make svc.AssumeRole request
// }
//
// func main() {
// sess := session.New()
// svc := sts.New(sess)
//
// myFunc(svc)
// }
//
// In your _test.go file:
//
// // Define a mock struct to be used in your unit tests of myFunc.
// type mockSTSClient struct {
// stsiface.STSAPI
// }
// func (m *mockSTSClient) AssumeRole(input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) {
// // mock response/functionality
// }
//
// func TestMyFunc(t *testing.T) {
// // Setup Test
// mockSvc := &mockSTSClient{}
//
// myfunc(mockSvc)
//
// // Verify myFunc's functionality
// }
//
// It is important to note that this interface will have breaking changes
// when the service model is updated and adds new API operations, paginators,
// and waiters. Its suggested to use the pattern above for testing, or using
// tooling to generate mocks to satisfy the interfaces.
type STSAPI interface {
AssumeRole(*sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error)
AssumeRoleWithContext(aws.Context, *sts.AssumeRoleInput, ...request.Option) (*sts.AssumeRoleOutput, error)
AssumeRoleRequest(*sts.AssumeRoleInput) (*request.Request, *sts.AssumeRoleOutput)
AssumeRoleWithSAML(*sts.AssumeRoleWithSAMLInput) (*sts.AssumeRoleWithSAMLOutput, error)
AssumeRoleWithSAMLWithContext(aws.Context, *sts.AssumeRoleWithSAMLInput, ...request.Option) (*sts.AssumeRoleWithSAMLOutput, error)
AssumeRoleWithSAMLRequest(*sts.AssumeRoleWithSAMLInput) (*request.Request, *sts.AssumeRoleWithSAMLOutput)
AssumeRoleWithWebIdentity(*sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error)
AssumeRoleWithWebIdentityWithContext(aws.Context, *sts.AssumeRoleWithWebIdentityInput, ...request.Option) (*sts.AssumeRoleWithWebIdentityOutput, error)
AssumeRoleWithWebIdentityRequest(*sts.AssumeRoleWithWebIdentityInput) (*request.Request, *sts.AssumeRoleWithWebIdentityOutput)
DecodeAuthorizationMessage(*sts.DecodeAuthorizationMessageInput) (*sts.DecodeAuthorizationMessageOutput, error)
DecodeAuthorizationMessageWithContext(aws.Context, *sts.DecodeAuthorizationMessageInput, ...request.Option) (*sts.DecodeAuthorizationMessageOutput, error)
DecodeAuthorizationMessageRequest(*sts.DecodeAuthorizationMessageInput) (*request.Request, *sts.DecodeAuthorizationMessageOutput)
GetAccessKeyInfo(*sts.GetAccessKeyInfoInput) (*sts.GetAccessKeyInfoOutput, error)
GetAccessKeyInfoWithContext(aws.Context, *sts.GetAccessKeyInfoInput, ...request.Option) (*sts.GetAccessKeyInfoOutput, error)
GetAccessKeyInfoRequest(*sts.GetAccessKeyInfoInput) (*request.Request, *sts.GetAccessKeyInfoOutput)
GetCallerIdentity(*sts.GetCallerIdentityInput) (*sts.GetCallerIdentityOutput, error)
GetCallerIdentityWithContext(aws.Context, *sts.GetCallerIdentityInput, ...request.Option) (*sts.GetCallerIdentityOutput, error)
GetCallerIdentityRequest(*sts.GetCallerIdentityInput) (*request.Request, *sts.GetCallerIdentityOutput)
GetFederationToken(*sts.GetFederationTokenInput) (*sts.GetFederationTokenOutput, error)
GetFederationTokenWithContext(aws.Context, *sts.GetFederationTokenInput, ...request.Option) (*sts.GetFederationTokenOutput, error)
GetFederationTokenRequest(*sts.GetFederationTokenInput) (*request.Request, *sts.GetFederationTokenOutput)
GetSessionToken(*sts.GetSessionTokenInput) (*sts.GetSessionTokenOutput, error)
GetSessionTokenWithContext(aws.Context, *sts.GetSessionTokenInput, ...request.Option) (*sts.GetSessionTokenOutput, error)
GetSessionTokenRequest(*sts.GetSessionTokenInput) (*request.Request, *sts.GetSessionTokenOutput)
}
var _ STSAPI = (*sts.STS)(nil)

10
vendor/github.com/goftp/server/.drone.yml generated vendored Normal file
View file

@ -0,0 +1,10 @@
kind: pipeline
name: default
steps:
- name: test
image: golang:1.12
commands:
- go get -t -d -v ./...
- go build -v
- go test -v -race -coverprofile=coverage.txt -covermode=atomic

View file

@ -4,6 +4,10 @@
package server
import (
"crypto/subtle"
)
// Auth is an interface to auth your ftp user login.
type Auth interface {
CheckPasswd(string, string) (bool, error)
@ -21,8 +25,9 @@ type SimpleAuth struct {
// CheckPasswd will check user's password
func (a *SimpleAuth) CheckPasswd(name, pass string) (bool, error) {
if name != a.Name || pass != a.Password {
return false, nil
}
return true, nil
return constantTimeEquals(name, a.Name) && constantTimeEquals(pass, a.Password), nil
}
func constantTimeEquals(a, b string) bool {
return len(a) == len(b) && subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1
}

View file

@ -5,6 +5,7 @@
package server
import (
"encoding/binary"
"fmt"
"log"
"strconv"
@ -36,6 +37,7 @@ var (
"EPSV": commandEpsv{},
"FEAT": commandFeat{},
"LIST": commandList{},
"LPRT": commandLprt{},
"NLST": commandNlst{},
"MDTM": commandMdtm{},
"MIC": commandMic{},
@ -63,6 +65,7 @@ var (
"USER": commandUser{},
"XCUP": commandCdup{},
"XCWD": commandCwd{},
"XMKD": commandMkd{},
"XPWD": commandPwd{},
"XRMD": commandRmd{},
}
@ -90,6 +93,8 @@ func (cmd commandAllo) Execute(conn *Conn, param string) {
conn.writeMessage(202, "Obsolete")
}
// commandAppe responds to the APPE FTP command. It allows the user to upload a
// new file but always append if file exists otherwise create one.
type commandAppe struct{}
func (cmd commandAppe) IsExtend() bool {
@ -97,7 +102,7 @@ func (cmd commandAppe) IsExtend() bool {
}
func (cmd commandAppe) RequireParam() bool {
return false
return true
}
func (cmd commandAppe) RequireAuth() bool {
@ -105,8 +110,16 @@ func (cmd commandAppe) RequireAuth() bool {
}
func (cmd commandAppe) Execute(conn *Conn, param string) {
conn.appendData = true
conn.writeMessage(202, "Obsolete")
targetPath := conn.buildPath(param)
conn.writeMessage(150, "Data transfer starting")
bytes, err := conn.driver.PutFile(targetPath, conn.dataConn, true)
if err == nil {
msg := "OK, received " + strconv.Itoa(int(bytes)) + " bytes"
conn.writeMessage(226, msg)
} else {
conn.writeMessage(450, fmt.Sprint("error during transfer: ", err))
}
}
type commandOpts struct{}
@ -283,6 +296,69 @@ func (cmd commandEprt) Execute(conn *Conn, param string) {
conn.writeMessage(200, "Connection established ("+strconv.Itoa(port)+")")
}
// commandLprt responds to the LPRT FTP command. It allows the client to
// request an active data socket with more options than the original PORT
// command. FTP Operation Over Big Address Records.
type commandLprt struct{}
func (cmd commandLprt) IsExtend() bool {
return true
}
func (cmd commandLprt) RequireParam() bool {
return true
}
func (cmd commandLprt) RequireAuth() bool {
return true
}
func (cmd commandLprt) Execute(conn *Conn, param string) {
// No tests for this code yet
parts := strings.Split(param, ",")
addressFamily, err := strconv.Atoi(parts[0])
if addressFamily != 4 {
conn.writeMessage(522, "Network protocol not supported, use 4")
return
}
addressLength, err := strconv.Atoi(parts[1])
if addressLength != 4 {
conn.writeMessage(522, "Network IP length not supported, use 4")
return
}
host := strings.Join(parts[2:2+addressLength], ".")
portLength, err := strconv.Atoi(parts[2+addressLength])
portAddress := parts[3+addressLength : 3+addressLength+portLength]
// Convert string[] to byte[]
portBytes := make([]byte, portLength)
for i := range portAddress {
p, _ := strconv.Atoi(portAddress[i])
portBytes[i] = byte(p)
}
// convert the bytes to an int
port := int(binary.BigEndian.Uint16(portBytes))
// if the existing connection is on the same host/port don't reconnect
if conn.dataConn.Host() == host && conn.dataConn.Port() == port {
return
}
socket, err := newActiveSocket(host, port, conn.logger, conn.sessionID)
if err != nil {
conn.writeMessage(425, "Data connection failed")
return
}
conn.dataConn = socket
conn.writeMessage(200, "Connection established ("+strconv.Itoa(port)+")")
}
// commandEpsv responds to the EPSV FTP command. It allows the client to
// request a passive data socket with more options than the original PASV
// command. It mainly adds ipv6 support, although we don't support that yet.
@ -302,13 +378,7 @@ func (cmd commandEpsv) RequireAuth() bool {
func (cmd commandEpsv) Execute(conn *Conn, param string) {
addr := conn.passiveListenIP()
lastIdx := strings.LastIndex(addr, ":")
if lastIdx <= 0 {
conn.writeMessage(425, "Data connection failed")
return
}
socket, err := newPassiveSocket(addr[:lastIdx], conn.PassivePort, conn.logger, conn.sessionID, conn.tlsConfig)
socket, err := newPassiveSocket(addr, conn.PassivePort, conn.logger, conn.sessionID, conn.tlsConfig)
if err != nil {
log.Println(err)
conn.writeMessage(425, "Data connection failed")
@ -577,12 +647,7 @@ func (cmd commandPasv) RequireAuth() bool {
func (cmd commandPasv) Execute(conn *Conn, param string) {
listenIP := conn.passiveListenIP()
lastIdx := strings.LastIndex(listenIP, ":")
if lastIdx <= 0 {
conn.writeMessage(425, "Data connection failed")
return
}
socket, err := newPassiveSocket(listenIP[:lastIdx], conn.PassivePort, conn.logger, conn.sessionID, conn.tlsConfig)
socket, err := newPassiveSocket(listenIP, conn.PassivePort, conn.logger, conn.sessionID, conn.tlsConfig)
if err != nil {
conn.writeMessage(425, "Data connection failed")
return
@ -590,7 +655,7 @@ func (cmd commandPasv) Execute(conn *Conn, param string) {
conn.dataConn = socket
p1 := socket.Port() / 256
p2 := socket.Port() - (p1 * 256)
quads := strings.Split(listenIP[:lastIdx], ".")
quads := strings.Split(listenIP, ".")
target := fmt.Sprintf("(%s,%s,%s,%s,%d,%d)", quads[0], quads[1], quads[2], quads[3], p1, p2)
msg := "Entering Passive Mode " + target
conn.writeMessage(227, msg)

View file

@ -58,10 +58,18 @@ func (conn *Conn) PublicIp() string {
}
func (conn *Conn) passiveListenIP() string {
var listenIP string
if len(conn.PublicIp()) > 0 {
return conn.PublicIp()
listenIP = conn.PublicIp()
} else {
listenIP = conn.conn.LocalAddr().(*net.TCPAddr).IP.String()
}
return conn.conn.LocalAddr().String()
lastIdx := strings.LastIndex(listenIP, ":")
if lastIdx <= 0 {
return listenIP
}
return listenIP[:lastIdx]
}
func (conn *Conn) PassivePort() int {

View file

@ -38,7 +38,6 @@ package proto
import (
"fmt"
"log"
"os"
"reflect"
"sort"
"strconv"
@ -194,7 +193,7 @@ func (p *Properties) Parse(s string) {
// "bytes,49,opt,name=foo,def=hello!"
fields := strings.Split(s, ",") // breaks def=, but handled below.
if len(fields) < 2 {
fmt.Fprintf(os.Stderr, "proto: tag has too few fields: %q\n", s)
log.Printf("proto: tag has too few fields: %q", s)
return
}
@ -214,7 +213,7 @@ func (p *Properties) Parse(s string) {
p.WireType = WireBytes
// no numeric converter for non-numeric types
default:
fmt.Fprintf(os.Stderr, "proto: tag has unknown wire type: %q\n", s)
log.Printf("proto: tag has unknown wire type: %q", s)
return
}

27
vendor/github.com/googleapis/gax-go/v2/LICENSE generated vendored Normal file
View file

@ -0,0 +1,27 @@
Copyright 2016, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

161
vendor/github.com/googleapis/gax-go/v2/call_option.go generated vendored Normal file
View file

@ -0,0 +1,161 @@
// Copyright 2016, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package gax
import (
"math/rand"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// CallOption is an option used by Invoke to control behaviors of RPC calls.
// CallOption works by modifying relevant fields of CallSettings.
type CallOption interface {
// Resolve applies the option by modifying cs.
Resolve(cs *CallSettings)
}
// Retryer is used by Invoke to determine retry behavior.
type Retryer interface {
// Retry reports whether a request should be retriedand how long to pause before retrying
// if the previous attempt returned with err. Invoke never calls Retry with nil error.
Retry(err error) (pause time.Duration, shouldRetry bool)
}
type retryerOption func() Retryer
func (o retryerOption) Resolve(s *CallSettings) {
s.Retry = o
}
// WithRetry sets CallSettings.Retry to fn.
func WithRetry(fn func() Retryer) CallOption {
return retryerOption(fn)
}
// OnCodes returns a Retryer that retries if and only if
// the previous attempt returns a GRPC error whose error code is stored in cc.
// Pause times between retries are specified by bo.
//
// bo is only used for its parameters; each Retryer has its own copy.
func OnCodes(cc []codes.Code, bo Backoff) Retryer {
return &boRetryer{
backoff: bo,
codes: append([]codes.Code(nil), cc...),
}
}
type boRetryer struct {
backoff Backoff
codes []codes.Code
}
func (r *boRetryer) Retry(err error) (time.Duration, bool) {
st, ok := status.FromError(err)
if !ok {
return 0, false
}
c := st.Code()
for _, rc := range r.codes {
if c == rc {
return r.backoff.Pause(), true
}
}
return 0, false
}
// Backoff implements exponential backoff.
// The wait time between retries is a random value between 0 and the "retry envelope".
// The envelope starts at Initial and increases by the factor of Multiplier every retry,
// but is capped at Max.
type Backoff struct {
// Initial is the initial value of the retry envelope, defaults to 1 second.
Initial time.Duration
// Max is the maximum value of the retry envelope, defaults to 30 seconds.
Max time.Duration
// Multiplier is the factor by which the retry envelope increases.
// It should be greater than 1 and defaults to 2.
Multiplier float64
// cur is the current retry envelope
cur time.Duration
}
// Pause returns the next time.Duration that the caller should use to backoff.
func (bo *Backoff) Pause() time.Duration {
if bo.Initial == 0 {
bo.Initial = time.Second
}
if bo.cur == 0 {
bo.cur = bo.Initial
}
if bo.Max == 0 {
bo.Max = 30 * time.Second
}
if bo.Multiplier < 1 {
bo.Multiplier = 2
}
// Select a duration between 1ns and the current max. It might seem
// counterintuitive to have so much jitter, but
// https://www.awsarchitectureblog.com/2015/03/backoff.html argues that
// that is the best strategy.
d := time.Duration(1 + rand.Int63n(int64(bo.cur)))
bo.cur = time.Duration(float64(bo.cur) * bo.Multiplier)
if bo.cur > bo.Max {
bo.cur = bo.Max
}
return d
}
type grpcOpt []grpc.CallOption
func (o grpcOpt) Resolve(s *CallSettings) {
s.GRPC = o
}
// WithGRPCOptions allows passing gRPC call options during client creation.
func WithGRPCOptions(opt ...grpc.CallOption) CallOption {
return grpcOpt(append([]grpc.CallOption(nil), opt...))
}
// CallSettings allow fine-grained control over how calls are made.
type CallSettings struct {
// Retry returns a Retryer to be used to control retry logic of a method call.
// If Retry is nil or the returned Retryer is nil, the call will not be retried.
Retry func() Retryer
// CallOptions to be forwarded to GRPC.
GRPC []grpc.CallOption
}

39
vendor/github.com/googleapis/gax-go/v2/gax.go generated vendored Normal file
View file

@ -0,0 +1,39 @@
// Copyright 2016, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Package gax contains a set of modules which aid the development of APIs
// for clients and servers based on gRPC and Google API conventions.
//
// Application code will rarely need to use this library directly.
// However, code generated automatically from API definition files can use it
// to simplify code generation and to provide more convenient and idiomatic API surfaces.
package gax
// Version specifies the gax-go version being used.
const Version = "2.0.4"

3
vendor/github.com/googleapis/gax-go/v2/go.mod generated vendored Normal file
View file

@ -0,0 +1,3 @@
module github.com/googleapis/gax-go/v2
require google.golang.org/grpc v1.19.0

25
vendor/github.com/googleapis/gax-go/v2/go.sum generated vendored Normal file
View file

@ -0,0 +1,25 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d h1:g9qWBGx4puODJTMVyoPrpoxPFgVGd+z1DZwjfRu4d0I=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

53
vendor/github.com/googleapis/gax-go/v2/header.go generated vendored Normal file
View file

@ -0,0 +1,53 @@
// Copyright 2018, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package gax
import "bytes"
// XGoogHeader is for use by the Google Cloud Libraries only.
//
// XGoogHeader formats key-value pairs.
// The resulting string is suitable for x-goog-api-client header.
func XGoogHeader(keyval ...string) string {
if len(keyval) == 0 {
return ""
}
if len(keyval)%2 != 0 {
panic("gax.Header: odd argument count")
}
var buf bytes.Buffer
for i := 0; i < len(keyval); i += 2 {
buf.WriteByte(' ')
buf.WriteString(keyval[i])
buf.WriteByte('/')
buf.WriteString(keyval[i+1])
}
return buf.String()[1:]
}

99
vendor/github.com/googleapis/gax-go/v2/invoke.go generated vendored Normal file
View file

@ -0,0 +1,99 @@
// Copyright 2016, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package gax
import (
"context"
"strings"
"time"
)
// APICall is a user defined call stub.
type APICall func(context.Context, CallSettings) error
// Invoke calls the given APICall,
// performing retries as specified by opts, if any.
func Invoke(ctx context.Context, call APICall, opts ...CallOption) error {
var settings CallSettings
for _, opt := range opts {
opt.Resolve(&settings)
}
return invoke(ctx, call, settings, Sleep)
}
// Sleep is similar to time.Sleep, but it can be interrupted by ctx.Done() closing.
// If interrupted, Sleep returns ctx.Err().
func Sleep(ctx context.Context, d time.Duration) error {
t := time.NewTimer(d)
select {
case <-ctx.Done():
t.Stop()
return ctx.Err()
case <-t.C:
return nil
}
}
type sleeper func(ctx context.Context, d time.Duration) error
// invoke implements Invoke, taking an additional sleeper argument for testing.
func invoke(ctx context.Context, call APICall, settings CallSettings, sp sleeper) error {
var retryer Retryer
for {
err := call(ctx, settings)
if err == nil {
return nil
}
if settings.Retry == nil {
return err
}
// Never retry permanent certificate errors. (e.x. if ca-certificates
// are not installed). We should only make very few, targeted
// exceptions: many (other) status=Unavailable should be retried, such
// as if there's a network hiccup, or the internet goes out for a
// minute. This is also why here we are doing string parsing instead of
// simply making Unavailable a non-retried code elsewhere.
if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
return err
}
if retryer == nil {
if r := settings.Retry(); r != nil {
retryer = r
} else {
return err
}
}
if d, ok := retryer.Retry(err); !ok {
return err
} else if err = sp(ctx, d); err != nil {
return err
}
}
}

View file

@ -73,6 +73,9 @@ func (c *LRU) Add(key, value interface{}) (evicted bool) {
func (c *LRU) Get(key interface{}) (value interface{}, ok bool) {
if ent, ok := c.items[key]; ok {
c.evictList.MoveToFront(ent)
if ent.Value.(*entry) == nil {
return nil, false
}
return ent.Value.(*entry).value, true
}
return
@ -142,6 +145,19 @@ func (c *LRU) Len() int {
return c.evictList.Len()
}
// Resize changes the cache size.
func (c *LRU) Resize(size int) (evicted int) {
diff := c.Len() - size
if diff < 0 {
diff = 0
}
for i := 0; i < diff; i++ {
c.removeOldest()
}
c.size = size
return diff
}
// removeOldest removes the oldest item from the cache.
func (c *LRU) removeOldest() {
ent := c.evictList.Back()

View file

@ -10,7 +10,7 @@ type LRUCache interface {
// updates the "recently used"-ness of the key. #value, isFound
Get(key interface{}) (value interface{}, ok bool)
// Check if a key exsists in cache without updating the recent-ness.
// Checks if a key exists in cache without updating the recent-ness.
Contains(key interface{}) (ok bool)
// Returns key's value without updating the "recently used"-ness of the key.
@ -31,6 +31,9 @@ type LRUCache interface {
// Returns the number of items in the cache.
Len() int
// Clear all cache entries
// Clears all cache entries.
Purge()
// Resizes cache, returning number evicted
Resize(int) int
}

View file

@ -32,3 +32,25 @@ if err := c.Quit(); err != nil {
log.Fatal(err)
}
```
## Store a file example ##
```go
data := bytes.NewBufferString("Hello World")
err = c.Stor("test-file.txt", data)
if err != nil {
panic(err)
}
```
## Read a file example ##
```go
r, err := c.Retr("test-file.txt")
if err != nil {
panic(err)
}
buf, err := ioutil.ReadAll(r)
println(string(buf))
```

View file

@ -73,7 +73,7 @@ type Response struct {
closed bool
}
// Dial connects to the specified address with optinal options
// Dial connects to the specified address with optional options
func Dial(addr string, options ...DialOption) (*ServerConn, error) {
do := &dialOptions{}
for _, option := range options {
@ -91,7 +91,7 @@ func Dial(addr string, options ...DialOption) (*ServerConn, error) {
if do.dialFunc != nil {
tconn, err = do.dialFunc("tcp", addr)
} else if do.tlsConfig != nil {
tconn, err = tls.DialWithDialer(&do.dialer , "tcp", addr, do.tlsConfig)
tconn, err = tls.DialWithDialer(&do.dialer, "tcp", addr, do.tlsConfig)
} else {
ctx := do.context
@ -172,7 +172,7 @@ func DialWithDisabledEPSV(disabled bool) DialOption {
}
// DialWithLocation returns a DialOption that configures the ServerConn with specified time.Location
// The lococation is used to parse the dates sent by the server which are in server's timezone
// The location is used to parse the dates sent by the server which are in server's timezone
func DialWithLocation(location *time.Location) DialOption {
return DialOption{func(do *dialOptions) {
do.location = location
@ -188,7 +188,7 @@ func DialWithContext(ctx context.Context) DialOption {
}
// DialWithTLS returns a DialOption that configures the ServerConn with specified TLS config
//
//
// If called together with the DialWithDialFunc option, the DialWithDialFunc function
// will be used when dialing new connections but regardless of the function,
// the connection will be treated as a TLS connection.

View file

@ -26,6 +26,7 @@ The tool is sponsored by the [marvin + konsorten GmbH](http://www.konsorten.de).
We thank all the authors who provided code to this library:
* Felix Kollmann
* Nicolas Perraut
## License

View file

@ -0,0 +1,11 @@
// +build linux darwin
package sequences
import (
"fmt"
)
func EnableVirtualTerminalProcessing(stream uintptr, enable bool) error {
return fmt.Errorf("windows only package")
}

View file

@ -17,11 +17,15 @@ import (
var XmlHeaderBytes []byte = []byte(xml.Header)
type ErrorHandlerFunc func(*http.Response, error) error
type PostHookFunc func(*http.Request, *http.Response) error
type HTTPClient struct {
BaseURL *url.URL
Headers http.Header
Client *http.Client
PostHooks map[int]func(*http.Request, *http.Response) error
PostHooks map[int]PostHookFunc
errorHandler ErrorHandlerFunc
rateLimited bool
rateLimitChan chan struct{}
rateLimitTimeout time.Duration
@ -31,7 +35,7 @@ func New() (httpClient *HTTPClient) {
return &HTTPClient{
Client: HttpClient,
Headers: make(http.Header),
PostHooks: make(map[int]func(*http.Request, *http.Response) error),
PostHooks: make(map[int]PostHookFunc),
}
}
@ -43,10 +47,14 @@ func Insecure() (httpClient *HTTPClient) {
var DefaultClient = New()
func (c *HTTPClient) SetPostHook(onStatus int, hook func(*http.Request, *http.Response) error) {
func (c *HTTPClient) SetPostHook(onStatus int, hook PostHookFunc) {
c.PostHooks[onStatus] = hook
}
func (c *HTTPClient) SetErrorHandler(handler ErrorHandlerFunc) {
c.errorHandler = handler
}
func (c *HTTPClient) SetRateLimit(limit int, timeout time.Duration) {
c.rateLimited = true
c.rateLimitChan = make(chan struct{}, limit)
@ -279,6 +287,10 @@ func (c *HTTPClient) Request(req *RequestData) (response *http.Response, err err
return nil, err
}
if req.Context != nil {
r = r.WithContext(req.Context)
}
r.ContentLength = req.ReqContentLength
if req.FullURL == "" {
@ -325,15 +337,27 @@ func (c *HTTPClient) Request(req *RequestData) (response *http.Response, err err
response, err = c.Client.Do(r)
}
if err != nil {
if req.Context != nil {
// If we got an error, and the context has been canceled,
// the context's error is probably more useful.
select {
case <-req.Context.Done():
err = req.Context.Err()
default:
}
}
if c.errorHandler != nil {
err = c.errorHandler(response, err)
}
return nil, err
}
if isTraceEnabled {
responseBytes, _ := httputil.DumpResponse(response, true)
fmt.Println(string(responseBytes))
}
if err != nil {
return response, err
}
if err = c.runPostHook(r, response); err != nil {
return response, err
}

View file

@ -1,6 +1,7 @@
package httpclient
import (
"context"
"io"
"net/http"
"net/url"
@ -15,6 +16,7 @@ const (
)
type RequestData struct {
Context context.Context
Method string
Path string
Params url.Values

View file

@ -23,17 +23,22 @@ func NewKoofrClient(baseUrl string, disableSecurity bool) *KoofrClient {
httpClient = httpclient.New()
}
return NewKoofrClientWithHTTPClient(baseUrl, httpClient)
}
func NewKoofrClientWithHTTPClient(baseUrl string, httpClient *httpclient.HTTPClient) *KoofrClient {
apiBaseUrl, _ := url.Parse(baseUrl)
httpClient.BaseURL = apiBaseUrl
httpClient.Headers.Set("User-Agent", "go koofrclient")
return &KoofrClient{
client:= &KoofrClient{
HTTPClient: httpClient,
token: "",
userID: "",
}
client.SetUserAgent("go koofrclient")
return client
}
func (c *KoofrClient) SetUserAgent(ua string) {

1
vendor/github.com/mattn/go-ieproxy/.gitignore generated vendored Normal file
View file

@ -0,0 +1 @@
.idea/*

11
vendor/github.com/mattn/go-ieproxy/GetProxyFunc.go generated vendored Normal file
View file

@ -0,0 +1,11 @@
package ieproxy
import (
"net/http"
"net/url"
)
// GetProxyFunc is a forwarder for the OS-Exclusive proxyMiddleman_os.go files
func GetProxyFunc() func(*http.Request) (*url.URL, error) {
return proxyMiddleman()
}

23
vendor/github.com/mattn/go-ieproxy/LICENSE generated vendored Normal file
View file

@ -0,0 +1,23 @@
MIT License
Copyright (c) 2014 mattn
Copyright (c) 2017 oliverpool
Copyright (c) 2019 Adele Reed
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

49
vendor/github.com/mattn/go-ieproxy/README.md generated vendored Normal file
View file

@ -0,0 +1,49 @@
# ieproxy
Go package to detect the proxy settings on Windows platform.
The settings are initially attempted to be read from the [`WinHttpGetIEProxyConfigForCurrentUser` DLL call](https://docs.microsoft.com/en-us/windows/desktop/api/winhttp/nf-winhttp-winhttpgetieproxyconfigforcurrentuser), but falls back to the registry (`CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Internet Settings`) in the event the DLL call fails.
For more information, take a look at the [documentation](https://godoc.org/github.com/mattn/go-ieproxy)
## Methods
You can either obtain a `net/http` compatible proxy function using `ieproxy.GetProxyFunc()`, set environment variables using `ieproxy.OverrideEnvWithStaticProxy()` (though no automatic configuration is available this way), or obtain the proxy settings via `ieproxy.GetConf()`.
| Method | Supported configuration options: |
|----------------------------------------|-----------------------------------------------|
| `ieproxy.GetProxyFunc()` | Static, Specified script, and fully automatic |
| `ieproxy.OverrideEnvWithStaticProxy()` | Static |
| `ieproxy.GetConf()` | Depends on how you use it |
## Examples
### Using GetProxyFunc():
```go
func init() {
http.DefaultTransport.(*http.Transport).Proxy = ieproxy.GetProxyFunc()
}
```
GetProxyFunc acts as a middleman between `net/http` and `mattn/go-ieproxy` in order to select the correct proxy configuration based off the details supplied in the config.
### Using OverrideEnvWithStaticProxy():
```go
func init() {
ieproxy.OverrideEnvWithStaticProxy()
http.DefaultTransport.(*http.Transport).Proxy = http.ProxyFromEnvironment
}
```
OverrideEnvWithStaticProxy overrides the relevant environment variables (`HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`) with the **static, manually configured** proxy details typically found in the registry.
### Using GetConf():
```go
func main() {
conf := ieproxy.GetConf()
//Handle proxies how you want to.
}
```

51
vendor/github.com/mattn/go-ieproxy/ieproxy.go generated vendored Normal file
View file

@ -0,0 +1,51 @@
// Package ieproxy is a utility to retrieve the proxy parameters (especially of Internet Explorer on windows)
//
// On windows, it gathers the parameters from the registry (regedit), while it uses env variable on other platforms
package ieproxy
import "os"
// ProxyConf gathers the configuration for proxy
type ProxyConf struct {
Static StaticProxyConf // static configuration
Automatic ProxyScriptConf // script configuration
}
// StaticProxyConf contains the configuration for static proxy
type StaticProxyConf struct {
// Is the proxy active?
Active bool
// Proxy address for each scheme (http, https)
// "" (empty string) is the fallback proxy
Protocols map[string]string
// Addresses not to be browsed via the proxy (comma-separated, linux-like)
NoProxy string
}
// ProxyScriptConf contains the configuration for automatic proxy
type ProxyScriptConf struct {
// Is the proxy active?
Active bool
// PreConfiguredURL of the .pac file.
// If this is empty and Active is true, auto-configuration should be assumed.
PreConfiguredURL string
}
// GetConf retrieves the proxy configuration from the Windows Regedit
func GetConf() ProxyConf {
return getConf()
}
// OverrideEnvWithStaticProxy writes new values to the
// `http_proxy`, `https_proxy` and `no_proxy` environment variables.
// The values are taken from the Windows Regedit (should be called in `init()` function - see example)
func OverrideEnvWithStaticProxy() {
overrideEnvWithStaticProxy(GetConf(), os.Setenv)
}
// FindProxyForURL computes the proxy for a given URL according to the pac file
func (psc *ProxyScriptConf) FindProxyForURL(URL string) string {
return psc.findProxyForURL(URL)
}
type envSetter func(string, string) error

10
vendor/github.com/mattn/go-ieproxy/ieproxy_unix.go generated vendored Normal file
View file

@ -0,0 +1,10 @@
// +build !windows
package ieproxy
func getConf() ProxyConf {
return ProxyConf{}
}
func overrideEnvWithStaticProxy(pc ProxyConf, setenv envSetter) {
}

164
vendor/github.com/mattn/go-ieproxy/ieproxy_windows.go generated vendored Normal file
View file

@ -0,0 +1,164 @@
package ieproxy
import (
"strings"
"sync"
"unsafe"
"golang.org/x/sys/windows/registry"
)
type regeditValues struct {
ProxyServer string
ProxyOverride string
ProxyEnable uint64
AutoConfigURL string
}
var once sync.Once
var windowsProxyConf ProxyConf
// GetConf retrieves the proxy configuration from the Windows Regedit
func getConf() ProxyConf {
once.Do(writeConf)
return windowsProxyConf
}
func writeConf() {
var (
cfg *tWINHTTP_CURRENT_USER_IE_PROXY_CONFIG
err error
)
if cfg, err = getUserConfigFromWindowsSyscall(); err != nil {
regedit, _ := readRegedit() // If the syscall fails, backup to manual detection.
windowsProxyConf = parseRegedit(regedit)
return
}
defer globalFreeWrapper(cfg.lpszProxy)
defer globalFreeWrapper(cfg.lpszProxyBypass)
defer globalFreeWrapper(cfg.lpszAutoConfigUrl)
windowsProxyConf = ProxyConf{
Static: StaticProxyConf{
Active: cfg.lpszProxy != nil,
},
Automatic: ProxyScriptConf{
Active: cfg.lpszAutoConfigUrl != nil || cfg.fAutoDetect,
},
}
if windowsProxyConf.Static.Active {
protocol := make(map[string]string)
for _, s := range strings.Split(StringFromUTF16Ptr(cfg.lpszProxy), ";") {
s = strings.TrimSpace(s)
if s == "" {
continue
}
pair := strings.SplitN(s, "=", 2)
if len(pair) > 1 {
protocol[pair[0]] = pair[1]
} else {
protocol[""] = pair[0]
}
}
windowsProxyConf.Static.Protocols = protocol
if cfg.lpszProxyBypass != nil {
windowsProxyConf.Static.NoProxy = strings.Replace(StringFromUTF16Ptr(cfg.lpszProxyBypass), ";", ",", -1)
}
}
if windowsProxyConf.Automatic.Active {
windowsProxyConf.Automatic.PreConfiguredURL = StringFromUTF16Ptr(cfg.lpszAutoConfigUrl)
}
}
func getUserConfigFromWindowsSyscall() (*tWINHTTP_CURRENT_USER_IE_PROXY_CONFIG, error) {
handle, _, err := winHttpOpen.Call(0, 0, 0, 0, 0)
if handle == 0 {
return &tWINHTTP_CURRENT_USER_IE_PROXY_CONFIG{}, err
}
defer winHttpCloseHandle.Call(handle)
config := new(tWINHTTP_CURRENT_USER_IE_PROXY_CONFIG)
ret, _, err := winHttpGetIEProxyConfigForCurrentUser.Call(uintptr(unsafe.Pointer(config)))
if ret > 0 {
err = nil
}
return config, err
}
// OverrideEnvWithStaticProxy writes new values to the
// http_proxy, https_proxy and no_proxy environment variables.
// The values are taken from the Windows Regedit (should be called in init() function)
func overrideEnvWithStaticProxy(conf ProxyConf, setenv envSetter) {
if conf.Static.Active {
for _, scheme := range []string{"http", "https"} {
url := mapFallback(scheme, "", conf.Static.Protocols)
setenv(scheme+"_proxy", url)
}
if conf.Static.NoProxy != "" {
setenv("no_proxy", conf.Static.NoProxy)
}
}
}
func parseRegedit(regedit regeditValues) ProxyConf {
protocol := make(map[string]string)
for _, s := range strings.Split(regedit.ProxyServer, ";") {
if s == "" {
continue
}
pair := strings.SplitN(s, "=", 2)
if len(pair) > 1 {
protocol[pair[0]] = pair[1]
} else {
protocol[""] = pair[0]
}
}
return ProxyConf{
Static: StaticProxyConf{
Active: regedit.ProxyEnable > 0,
Protocols: protocol,
NoProxy: strings.Replace(regedit.ProxyOverride, ";", ",", -1), // to match linux style
},
Automatic: ProxyScriptConf{
Active: regedit.AutoConfigURL != "",
PreConfiguredURL: regedit.AutoConfigURL,
},
}
}
func readRegedit() (values regeditValues, err error) {
k, err := registry.OpenKey(registry.CURRENT_USER, `Software\Microsoft\Windows\CurrentVersion\Internet Settings`, registry.QUERY_VALUE)
if err != nil {
return
}
defer k.Close()
values.ProxyServer, _, err = k.GetStringValue("ProxyServer")
if err != nil && err != registry.ErrNotExist {
return
}
values.ProxyOverride, _, err = k.GetStringValue("ProxyOverride")
if err != nil && err != registry.ErrNotExist {
return
}
values.ProxyEnable, _, err = k.GetIntegerValue("ProxyEnable")
if err != nil && err != registry.ErrNotExist {
return
}
values.AutoConfigURL, _, err = k.GetStringValue("AutoConfigURL")
if err != nil && err != registry.ErrNotExist {
return
}
err = nil
return
}

Some files were not shown because too many files have changed in this diff Show more