vendor: update all dependencies

This commit is contained in:
Nick Craig-Wood 2017-07-23 08:51:42 +01:00
parent 0b6fba34a3
commit eb87cf6f12
2008 changed files with 352633 additions and 1004750 deletions

View file

@ -3,6 +3,5 @@
server_standalone/server_standalone
examples/sftp-server/id_rsa
examples/sftp-server/id_rsa.pub
examples/sftp-server/sftp-server
examples/*/id_rsa
examples/*/id_rsa.pub

15
vendor/github.com/pkg/sftp/client.go generated vendored
View file

@ -653,7 +653,9 @@ func (f *File) Read(b []byte) (int, error) {
inFlight := 0
desiredInFlight := 1
offset := f.offset
ch := make(chan result, 2)
// maxConcurrentRequests buffer to deal with broadcastErr() floods
// also must have a buffer of max value of (desiredInFlight - inFlight)
ch := make(chan result, maxConcurrentRequests)
type inflightRead struct {
b []byte
offset uint64
@ -748,7 +750,8 @@ func (f *File) WriteTo(w io.Writer) (int64, error) {
offset := f.offset
writeOffset := offset
fileSize := uint64(fi.Size())
ch := make(chan result, 2)
// see comment on same line in Read() above
ch := make(chan result, maxConcurrentRequests)
type inflightRead struct {
b []byte
offset uint64
@ -890,8 +893,8 @@ func (f *File) Write(b []byte) (int, error) {
inFlight := 0
desiredInFlight := 1
offset := f.offset
// chan must have a buffer of max value of (desiredInFlight - inFlight)
ch := make(chan result, 2)
// see comment on same line in Read() above
ch := make(chan result, maxConcurrentRequests)
var firstErr error
written := len(b)
for len(b) > 0 || inFlight > 0 {
@ -951,8 +954,8 @@ func (f *File) ReadFrom(r io.Reader) (int64, error) {
inFlight := 0
desiredInFlight := 1
offset := f.offset
// chan must have a buffer of max value of (desiredInFlight - inFlight)
ch := make(chan result, 2)
// see comment on same line in Read() above
ch := make(chan result, maxConcurrentRequests)
var firstErr error
read := int64(0)
b := make([]byte, f.c.maxPacket)

View file

@ -92,7 +92,7 @@ func (w delayedWriter) Close() error {
// netPipe provides a pair of io.ReadWriteClosers connected to each other.
// The functions is identical to os.Pipe with the exception that netPipe
// provides the Read/Close guarentees that os.File derrived pipes do not.
// provides the Read/Close guarantees that os.File derrived pipes do not.
func netPipe(t testing.TB) (io.ReadWriteCloser, io.ReadWriteCloser) {
type result struct {
net.Conn
@ -1609,9 +1609,9 @@ func contains(vector []string, s string) bool {
var globTests = []struct {
pattern, result string
}{
{"match.go", "./match.go"},
{"mat?h.go", "./match.go"},
{"ma*ch.go", "./match.go"},
{"match.go", "match.go"},
{"mat?h.go", "match.go"},
{"ma*ch.go", "match.go"},
{"../*/match.go", "../sftp/match.go"},
}
@ -1711,6 +1711,35 @@ func TestServerRoughDisconnect(t *testing.T) {
io.Copy(ioutil.Discard, f)
}
// sftp/issue/181, abrupt server hangup would result in client hangs.
// due to broadcastErr filling up the request channel
// this reproduces it about 50% of the time
func TestServerRoughDisconnect2(t *testing.T) {
if *testServerImpl {
t.Skipf("skipping with -testserver")
}
sftp, cmd := testClient(t, READONLY, NO_DELAY)
defer cmd.Wait()
defer sftp.Close()
f, err := sftp.Open("/dev/zero")
if err != nil {
t.Fatal(err)
}
defer f.Close()
b := make([]byte, 32768*100)
go func() {
time.Sleep(1 * time.Millisecond)
cmd.Process.Kill()
}()
for {
_, err = f.Read(b)
if err != nil {
break
}
}
}
// sftp/issue/26 writing to a read only file caused client to loop.
func TestClientWriteToROFile(t *testing.T) {
sftp, cmd := testClient(t, READWRITE, NO_DELAY)
@ -1902,6 +1931,88 @@ func BenchmarkWrite4MiBDelay150Msec(b *testing.B) {
benchmarkWrite(b, 4*1024*1024, 150*time.Millisecond)
}
func benchmarkReadFrom(b *testing.B, bufsize int, delay time.Duration) {
size := 10*1024*1024 + 123 // ~10MiB
// open sftp client
sftp, cmd := testClient(b, false, delay)
defer cmd.Wait()
// defer sftp.Close()
data := make([]byte, size)
b.ResetTimer()
b.SetBytes(int64(size))
for i := 0; i < b.N; i++ {
f, err := ioutil.TempFile("", "sftptest")
if err != nil {
b.Fatal(err)
}
defer os.Remove(f.Name())
f2, err := sftp.Create(f.Name())
if err != nil {
b.Fatal(err)
}
defer f2.Close()
f2.ReadFrom(bytes.NewReader(data))
f2.Close()
fi, err := os.Stat(f.Name())
if err != nil {
b.Fatal(err)
}
if fi.Size() != int64(size) {
b.Fatalf("wrong file size: want %d, got %d", size, fi.Size())
}
os.Remove(f.Name())
}
}
func BenchmarkReadFrom1k(b *testing.B) {
benchmarkReadFrom(b, 1*1024, NO_DELAY)
}
func BenchmarkReadFrom16k(b *testing.B) {
benchmarkReadFrom(b, 16*1024, NO_DELAY)
}
func BenchmarkReadFrom32k(b *testing.B) {
benchmarkReadFrom(b, 32*1024, NO_DELAY)
}
func BenchmarkReadFrom128k(b *testing.B) {
benchmarkReadFrom(b, 128*1024, NO_DELAY)
}
func BenchmarkReadFrom512k(b *testing.B) {
benchmarkReadFrom(b, 512*1024, NO_DELAY)
}
func BenchmarkReadFrom1MiB(b *testing.B) {
benchmarkReadFrom(b, 1024*1024, NO_DELAY)
}
func BenchmarkReadFrom4MiB(b *testing.B) {
benchmarkReadFrom(b, 4*1024*1024, NO_DELAY)
}
func BenchmarkReadFrom4MiBDelay10Msec(b *testing.B) {
benchmarkReadFrom(b, 4*1024*1024, 10*time.Millisecond)
}
func BenchmarkReadFrom4MiBDelay50Msec(b *testing.B) {
benchmarkReadFrom(b, 4*1024*1024, 50*time.Millisecond)
}
func BenchmarkReadFrom4MiBDelay150Msec(b *testing.B) {
benchmarkReadFrom(b, 4*1024*1024, 150*time.Millisecond)
}
func benchmarkCopyDown(b *testing.B, fileSize int64, delay time.Duration) {
// Create a temp file and fill it with zero's.
src, err := ioutil.TempFile("", "sftptest")

View file

@ -5,6 +5,8 @@ import (
"log"
"os"
"os/exec"
"path"
"strings"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
@ -90,3 +92,44 @@ func ExampleNewClientPipe() {
// close the connection
client.Close()
}
func ExampleClient_Mkdir_parents() {
// Example of mimicing 'mkdir --parents'; I.E. recursively create
// directoryies and don't error if any directories already exists.
var conn *ssh.Client
client, err := sftp.NewClient(conn)
if err != nil {
log.Fatal(err)
}
defer client.Close()
ssh_fx_failure := uint32(4)
mkdirParents := func(client *sftp.Client, dir string) (err error) {
var parents string
for _, name := range strings.Split(dir, "/") {
parents = path.Join(parents, name)
err = client.Mkdir(parents)
if status, ok := err.(*sftp.StatusError); ok {
if status.Code == ssh_fx_failure {
var fi os.FileInfo
fi, err = client.Stat(parents)
if err == nil {
if !fi.IsDir() {
return fmt.Errorf("File exists: %s", parents)
}
}
}
}
if err != nil {
break
}
}
return err
}
err = mkdirParents(client, "/tmp/foo/bar")
if err != nil {
log.Fatal(err)
}
}

View file

@ -72,12 +72,12 @@ func main() {
log.Fatal("failed to accept incoming connection", err)
}
// Before use, a handshake must be performed on the incoming
// net.Conn.
_, chans, reqs, err := ssh.NewServerConn(nConn, config)
// Before use, a handshake must be performed on the incoming net.Conn.
sconn, chans, reqs, err := ssh.NewServerConn(nConn, config)
if err != nil {
log.Fatal("failed to handshake", err)
}
log.Println("login detected:", sconn.User())
fmt.Fprintf(debugStream, "SSH server established\n")
// The incoming Request channel must be serviced.

58
vendor/github.com/pkg/sftp/match.go generated vendored
View file

@ -1,13 +1,13 @@
package sftp
import (
"errors"
"path"
"strings"
"unicode/utf8"
)
// ErrBadPattern indicates a globbing pattern was malformed.
var ErrBadPattern = errors.New("syntax error in pattern")
var ErrBadPattern = path.ErrBadPattern
// Unix separator
const separator = "/"
@ -36,48 +36,7 @@ const separator = "/"
//
//
func Match(pattern, name string) (matched bool, err error) {
Pattern:
for len(pattern) > 0 {
var star bool
var chunk string
star, chunk, pattern = scanChunk(pattern)
if star && chunk == "" {
// Trailing * matches rest of string unless it has a /.
return !strings.Contains(name, separator), nil
}
// Look for match at current position.
t, ok, err := matchChunk(chunk, name)
// if we're the last chunk, make sure we've exhausted the name
// otherwise we'll give a false result even if we could still match
// using the star
if ok && (len(t) == 0 || len(pattern) > 0) {
name = t
continue
}
if err != nil {
return false, err
}
if star {
// Look for match skipping i+1 bytes.
// Cannot skip /.
for i := 0; i < len(name) && !isPathSeparator(name[i]); i++ {
t, ok, err := matchChunk(chunk, name[i+1:])
if ok {
// if we're the last chunk, make sure we exhausted the name
if len(pattern) == 0 && len(t) > 0 {
continue
}
name = t
continue Pattern
}
if err != nil {
return false, err
}
}
}
return false, nil
}
return len(name) == 0, nil
return path.Match(pattern, name)
}
// detect if byte(char) is path separator
@ -325,16 +284,7 @@ func (c *Client) glob(dir, pattern string, matches []string) (m []string, e erro
// a Separator if necessary.
// all empty strings are ignored.
func Join(elem ...string) string {
return join(elem)
}
func join(elem []string) string {
// If there's a bug here, fix the logic in ./path_plan9.go too.
for i, e := range elem {
if e != "" {
return strings.Join(elem[i:], string(separator))
}
}
return ""
return path.Join(elem...)
}
// hasMeta reports whether path contains any of the magic characters

View file

@ -61,15 +61,15 @@ func (s packetManager) close() {
// The goal is to process packets in the order they are received as is
// requires by section 7 of the RFC, while maximizing throughput of file
// transfers.
func (s *packetManager) workerChan(worker func(requestChan)) requestChan {
func (s *packetManager) workerChan(runWorker func(requestChan)) requestChan {
rwChan := make(chan requestPacket, sftpServerWorkerCount)
for i := 0; i < sftpServerWorkerCount; i++ {
go worker(rwChan)
runWorker(rwChan)
}
cmdChan := make(chan requestPacket)
go worker(cmdChan)
runWorker(cmdChan)
pktChan := make(chan requestPacket, sftpServerWorkerCount)
go func() {

View file

@ -2,6 +2,7 @@ package sftp
import (
"encoding"
"fmt"
"sync"
"testing"
"time"
@ -89,6 +90,19 @@ func TestPacketManager(t *testing.T) {
s.close()
}
func (p sshFxpRemovePacket) String() string {
return fmt.Sprintf("RmPct:%d", p.ID)
}
func (p sshFxpOpenPacket) String() string {
return fmt.Sprintf("OpPct:%d", p.ID)
}
func (p sshFxpWritePacket) String() string {
return fmt.Sprintf("WrPct:%d", p.ID)
}
func (p sshFxpClosePacket) String() string {
return fmt.Sprintf("ClPct:%d", p.ID)
}
// Test what happens when the pool processes a close packet on a file that it
// is still reading from.
func TestCloseOutOfOrder(t *testing.T) {
@ -108,18 +122,20 @@ func TestCloseOutOfOrder(t *testing.T) {
pktMgr := newPktMgr(sender)
wg := sync.WaitGroup{}
wg.Add(len(packets))
worker := func(ch requestChan) {
for pkt := range ch {
if _, ok := pkt.(*sshFxpWritePacket); ok {
// sleep to cause writes to come after close/remove
time.Sleep(time.Millisecond)
runWorker := func(ch requestChan) {
go func() {
for pkt := range ch {
if _, ok := pkt.(*sshFxpWritePacket); ok {
// sleep to cause writes to come after close/remove
time.Sleep(time.Millisecond)
}
pktMgr.working.Done()
recvChan <- pkt
wg.Done()
}
pktMgr.working.Done()
recvChan <- pkt
wg.Done()
}
}()
}
pktChan := pktMgr.workerChan(worker)
pktChan := pktMgr.workerChan(runWorker)
for _, p := range packets {
pktChan <- p
}

View file

@ -10,6 +10,8 @@ import (
"io"
"os"
"path/filepath"
"sort"
"strconv"
"sync"
"time"
)
@ -104,13 +106,35 @@ func (fs *root) Fileinfo(r Request) ([]os.FileInfo, error) {
defer fs.filesLock.Unlock()
switch r.Method {
case "List":
list := []os.FileInfo{}
for fn, fi := range fs.files {
if filepath.Dir(fn) == r.Filepath {
list = append(list, fi)
var err error
batch_size := 10
current_offset := 0
if token := r.LsNext(); token != "" {
current_offset, err = strconv.Atoi(token)
if err != nil {
return nil, os.ErrInvalid
}
}
return list, nil
ordered_names := []string{}
for fn, _ := range fs.files {
if filepath.Dir(fn) == r.Filepath {
ordered_names = append(ordered_names, fn)
}
}
sort.Sort(sort.StringSlice(ordered_names))
list := make([]os.FileInfo, len(ordered_names))
for i, fn := range ordered_names {
list[i] = fs.files[fn]
}
if len(list) < current_offset {
return nil, io.EOF
}
new_offset := current_offset + batch_size
if new_offset > len(list) {
new_offset = len(list)
}
r.LsSave(strconv.Itoa(new_offset))
return list[current_offset:new_offset], nil
case "Stat":
file, err := fs.fetch(r.Filepath)
if err != nil {

View file

@ -26,7 +26,7 @@ type Handlers struct {
// RequestServer abstracts the sftp protocol with an http request-like protocol
type RequestServer struct {
serverConn
*serverConn
Handlers Handlers
pktMgr packetManager
openRequests map[string]Request
@ -37,7 +37,7 @@ type RequestServer struct {
// NewRequestServer creates/allocates/returns new RequestServer.
// Normally there there will be one server per user-session.
func NewRequestServer(rwc io.ReadWriteCloser, h Handlers) *RequestServer {
svrConn := serverConn{
svrConn := &serverConn{
conn: conn{
Reader: rwc,
WriteCloser: rwc,
@ -46,7 +46,7 @@ func NewRequestServer(rwc io.ReadWriteCloser, h Handlers) *RequestServer {
return &RequestServer{
serverConn: svrConn,
Handlers: h,
pktMgr: newPktMgr(&svrConn),
pktMgr: newPktMgr(svrConn),
openRequests: make(map[string]Request),
}
}
@ -82,15 +82,16 @@ func (rs *RequestServer) Close() error { return rs.conn.Close() }
// Serve requests for user session
func (rs *RequestServer) Serve() error {
var wg sync.WaitGroup
wg.Add(1)
workerFunc := func(ch requestChan) {
runWorker := func(ch requestChan) {
wg.Add(1)
defer wg.Done()
if err := rs.packetWorker(ch); err != nil {
rs.conn.Close() // shuts down recvPacket
}
go func() {
defer wg.Done()
if err := rs.packetWorker(ch); err != nil {
rs.conn.Close() // shuts down recvPacket
}
}()
}
pktChan := rs.pktMgr.workerChan(workerFunc)
pktChan := rs.pktMgr.workerChan(runWorker)
var err error
var pkt requestPacket
@ -111,7 +112,6 @@ func (rs *RequestServer) Serve() error {
pktChan <- pkt
}
wg.Done()
close(pktChan) // shuts down sftpServerWorkers
wg.Wait() // wait for all workers to exit

View file

@ -5,7 +5,6 @@ import (
"io"
"net"
"os"
"sort"
"testing"
"github.com/stretchr/testify/assert"
@ -317,14 +316,14 @@ func TestRequestReadlink(t *testing.T) {
func TestRequestReaddir(t *testing.T) {
p := clientRequestServerPair(t)
defer p.Close()
_, err := putTestFile(p.cli, "/foo", "hello")
assert.Nil(t, err)
_, err = putTestFile(p.cli, "/bar", "goodbye")
assert.Nil(t, err)
for i := 0; i < 100; i++ {
fname := fmt.Sprintf("/foo_%02d", i)
_, err := putTestFile(p.cli, fname, fname)
assert.Nil(t, err)
}
di, err := p.cli.ReadDir("/")
assert.Nil(t, err)
assert.Len(t, di, 2)
names := []string{di[0].Name(), di[1].Name()}
sort.Strings(names)
assert.Equal(t, []string{"bar", "foo"}, names)
assert.Len(t, di, 100)
names := []string{di[18].Name(), di[81].Name()}
assert.Equal(t, []string{"foo_18", "foo_81"}, names)
}

View file

@ -29,9 +29,10 @@ type Request struct {
}
type state struct {
writerAt io.WriterAt
readerAt io.ReaderAt
endofdir bool // need to track when to send EOF for readdir
writerAt io.WriterAt
readerAt io.ReaderAt
endofdir bool // in case handler doesn't use EOF on file list
readdirToken string
}
type packet_data struct {
@ -67,8 +68,24 @@ func NewRequest(method, path string) Request {
return request
}
// manage state
func (r Request) setState(s interface{}) {
// LsSave takes a token to keep track of file list batches. Openssh uses a
// batch size of 100, so I suggest sticking close to that.
func (r Request) LsSave(token string) {
r.stateLock.RLock()
defer r.stateLock.RUnlock()
r.state.readdirToken = token
}
// LsNext should return the token from the previous call to know which batch
// to return next.
func (r Request) LsNext() string {
r.stateLock.RLock()
defer r.stateLock.RUnlock()
return r.state.readdirToken
}
// manage file read/write state
func (r Request) setFileState(s interface{}) {
r.stateLock.Lock()
defer r.stateLock.Unlock()
switch s := s.(type) {
@ -76,8 +93,7 @@ func (r Request) setState(s interface{}) {
r.state.writerAt = s
case io.ReaderAt:
r.state.readerAt = s
case bool:
r.state.endofdir = s
}
}
@ -93,6 +109,14 @@ func (r Request) getReader() io.ReaderAt {
return r.state.readerAt
}
// For backwards compatibility. The Handler didn't have batch handling at
// first, and just always assumed 1 batch. This preserves that behavior.
func (r Request) setEOD(eod bool) {
r.stateLock.RLock()
defer r.stateLock.RUnlock()
r.state.endofdir = eod
}
func (r Request) getEOD() bool {
r.stateLock.RLock()
defer r.stateLock.RUnlock()
@ -149,7 +173,7 @@ func fileget(h FileReader, r Request) (responsePacket, error) {
if err != nil {
return nil, err
}
r.setState(reader)
r.setFileState(reader)
}
pd := r.popPacket()
@ -174,7 +198,7 @@ func fileput(h FileWriter, r Request) (responsePacket, error) {
if err != nil {
return nil, err
}
r.setState(writer)
r.setFileState(writer)
}
pd := r.popPacket()
@ -224,7 +248,14 @@ func fileinfo(h FileInfoer, r Request) (responsePacket, error) {
Attrs: []interface{}{fi},
})
}
r.setState(true)
// No entries means we should return EOF as the Handler didn't.
if len(finfo) == 0 {
return nil, io.EOF
}
// If files are returned but no token is set, return EOF next call.
if r.LsNext() == "" {
r.setEOD(true)
}
return ret, nil
case "Stat":
if len(finfo) == 0 {

22
vendor/github.com/pkg/sftp/server.go generated vendored
View file

@ -26,7 +26,7 @@ const (
// This implementation currently supports most of sftp server protocol version 3,
// as specified at http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02
type Server struct {
serverConn
*serverConn
debugStream io.Writer
readOnly bool
pktMgr packetManager
@ -75,7 +75,7 @@ type serverRespondablePacket interface {
//
// A subsequent call to Serve() is required to begin serving files over SFTP.
func NewServer(rwc io.ReadWriteCloser, options ...ServerOption) (*Server, error) {
svrConn := serverConn{
svrConn := &serverConn{
conn: conn{
Reader: rwc,
WriteCloser: rwc,
@ -84,7 +84,7 @@ func NewServer(rwc io.ReadWriteCloser, options ...ServerOption) (*Server, error)
s := &Server{
serverConn: svrConn,
debugStream: ioutil.Discard,
pktMgr: newPktMgr(&svrConn),
pktMgr: newPktMgr(svrConn),
openFiles: make(map[string]*os.File),
maxTxPacket: 1 << 15,
}
@ -282,15 +282,16 @@ func handlePacket(s *Server, p interface{}) error {
// is stopped.
func (svr *Server) Serve() error {
var wg sync.WaitGroup
wg.Add(1)
workerFunc := func(ch requestChan) {
runWorker := func(ch requestChan) {
wg.Add(1)
defer wg.Done()
if err := svr.sftpServerWorker(ch); err != nil {
svr.conn.Close() // shuts down recvPacket
}
go func() {
defer wg.Done()
if err := svr.sftpServerWorker(ch); err != nil {
svr.conn.Close() // shuts down recvPacket
}
}()
}
pktChan := svr.pktMgr.workerChan(workerFunc)
pktChan := svr.pktMgr.workerChan(runWorker)
var err error
var pkt requestPacket
@ -311,7 +312,6 @@ func (svr *Server) Serve() error {
pktChan <- pkt
}
wg.Done()
close(pktChan) // shuts down sftpServerWorkers
wg.Wait() // wait for all workers to exit