serve s3: fixes before merge
- add context to log and fallthrough to error log level - test: use rclone random lib to generate random strings - calculate hash from vfs cache if file is uploading - add server started log with server url - remove md5 hasher
This commit is contained in:
Saw-jan
Nick Craig-Wood
parent
34ef5147aa
commit
d3dcc61154
8 changed files with 62 additions and 36 deletions
|
|
@ -3,7 +3,6 @@ package s3
|
|||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"log"
|
||||
|
|
@ -306,8 +305,7 @@ func (db *s3Backend) PutObject(
|
|||
return result, err
|
||||
}
|
||||
|
||||
hasher := md5.New()
|
||||
w := io.MultiWriter(f, hasher)
|
||||
w := io.MultiWriter(f)
|
||||
if _, err := io.Copy(w, input); err != nil {
|
||||
// remove file when i/o error occurred (FsPutErr)
|
||||
_ = f.Close()
|
||||
|
|
|
|||
|
|
@ -12,15 +12,14 @@ type logger struct{}
|
|||
|
||||
// print log message
|
||||
func (l logger) Print(level gofakes3.LogLevel, v ...interface{}) {
|
||||
// fs.Infof(nil, fmt.Sprintln(v...))
|
||||
switch level {
|
||||
case gofakes3.LogErr:
|
||||
fs.Errorf(nil, fmt.Sprintln(v...))
|
||||
case gofakes3.LogWarn:
|
||||
fs.Infof(nil, fmt.Sprintln(v...))
|
||||
case gofakes3.LogInfo:
|
||||
fs.Debugf(nil, fmt.Sprintln(v...))
|
||||
default:
|
||||
panic("unknown level")
|
||||
fallthrough
|
||||
case gofakes3.LogErr:
|
||||
fs.Errorf("serve s3", fmt.Sprintln(v...))
|
||||
case gofakes3.LogWarn:
|
||||
fs.Infof("serve s3", fmt.Sprintln(v...))
|
||||
case gofakes3.LogInfo:
|
||||
fs.Debugf("serve s3", fmt.Sprintln(v...))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -61,7 +61,10 @@ var Command = &cobra.Command{
|
|||
}
|
||||
router := s.Router()
|
||||
s.Bind(router)
|
||||
s.Serve()
|
||||
err = s.serve()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.Wait()
|
||||
return nil
|
||||
})
|
||||
|
|
|
|||
|
|
@ -6,10 +6,8 @@ package s3
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
|
@ -29,6 +27,7 @@ import (
|
|||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
httplib "github.com/rclone/rclone/lib/http"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -39,8 +38,8 @@ const (
|
|||
|
||||
// Configure and serve the server
|
||||
func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) {
|
||||
keyid = RandString(16)
|
||||
keysec = RandString(16)
|
||||
keyid = random.String(16)
|
||||
keysec = random.String(16)
|
||||
serveropt := &Options{
|
||||
HTTP: httplib.DefaultCfg(),
|
||||
pathBucketMode: true,
|
||||
|
|
@ -60,17 +59,6 @@ func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) {
|
|||
return
|
||||
}
|
||||
|
||||
func RandString(n int) string {
|
||||
src := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
b := make([]byte, (n+1)/2)
|
||||
|
||||
if _, err := src.Read(b); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return hex.EncodeToString(b)[:n]
|
||||
}
|
||||
|
||||
// TestS3 runs the s3 server then runs the unit tests for the
|
||||
// s3 remote against it.
|
||||
func TestS3(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -70,3 +70,9 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error
|
|||
func (w *Server) Bind(router chi.Router) {
|
||||
router.Handle("/*", w.handler)
|
||||
}
|
||||
|
||||
func (w *Server) serve() error {
|
||||
w.Serve()
|
||||
fs.Logf(w.f, "Starting s3 server on %s", w.URLs())
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,11 +4,14 @@ import (
|
|||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/Mikubill/gofakes3"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
)
|
||||
|
||||
|
|
@ -47,9 +50,29 @@ func getFileHash(node interface{}) string {
|
|||
|
||||
switch b := node.(type) {
|
||||
case vfs.Node:
|
||||
o = b.DirEntry().(fs.Object)
|
||||
case fs.DirEntry:
|
||||
o = b.(fs.Object)
|
||||
fsObj, ok := b.DirEntry().(fs.Object)
|
||||
if !ok {
|
||||
fs.Debugf("serve s3", "File uploading - reading hash from VFS cache")
|
||||
in, err := b.Open(os.O_RDONLY)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer func() {
|
||||
_ = in.Close()
|
||||
}()
|
||||
h, err := hash.NewMultiHasherTypes(hash.NewHashSet(Opt.hashType))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
_, err = io.Copy(h, in)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return h.Sums()[Opt.hashType]
|
||||
}
|
||||
o = fsObj
|
||||
case fs.Object:
|
||||
o = b
|
||||
}
|
||||
|
||||
hash, err := o.Hash(context.Background(), Opt.hashType)
|
||||
|
|
|
|||
7
go.mod
7
go.mod
|
|
@ -43,7 +43,7 @@ require (
|
|||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6
|
||||
github.com/mattn/go-colorable v0.1.13
|
||||
github.com/mattn/go-runewidth v0.0.15
|
||||
github.com/minio/minio-go/v7 v7.0.57
|
||||
github.com/minio/minio-go/v7 v7.0.63
|
||||
github.com/mitchellh/go-homedir v1.1.0
|
||||
github.com/moby/sys/mountinfo v0.6.2
|
||||
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1
|
||||
|
|
@ -131,6 +131,7 @@ require (
|
|||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 // indirect
|
||||
github.com/jtolio/noiseconn v0.0.0-20230621152802-afeab29449e0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
|
||||
|
|
@ -140,6 +141,10 @@ require (
|
|||
github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
|
|
|
|||
14
go.sum
14
go.sum
|
|
@ -58,8 +58,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIf
|
|||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
|
||||
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
|
||||
github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e h1:gtBhC9D1R/uuoov9wO8IDx3E25Tqn8nW7xRTvgPDP2E=
|
||||
github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY=
|
||||
github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 h1:r3fp2/Ro+0RtpjNY0/wsbN7vRmCW//dXTOZDQTct25Q=
|
||||
github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY=
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I=
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug=
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo=
|
||||
|
|
@ -355,6 +355,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
|
|||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
|
||||
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
|
|
@ -396,8 +397,8 @@ github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0
|
|||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
|
||||
github.com/minio/minio-go/v7 v7.0.57 h1:xsFiOiWjpC1XAGbFEUOzj1/gMXGz7ljfxifwcb/5YXU=
|
||||
github.com/minio/minio-go/v7 v7.0.57/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE=
|
||||
github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ=
|
||||
github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4=
|
||||
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
|
||||
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
|
||||
|
|
@ -469,10 +470,14 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
|
|||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
|
||||
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
|
||||
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
|
||||
github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI=
|
||||
github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0=
|
||||
github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E=
|
||||
github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA=
|
||||
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
|
||||
|
|
@ -823,7 +828,6 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
|
|||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
|
|
|
|||
Loading…
Reference in a new issue