From 8c8ee9905c6435050873f7724a080d8f49e100c5 Mon Sep 17 00:00:00 2001 From: Manoj Ghosh Date: Wed, 25 Jan 2023 02:48:16 -0800 Subject: [PATCH] oracleobjectstorage: speed up operations by using S3 pacer and setting minsleep to 10ms Uploading 100 files of each 1 MB took 20 seconds before. With above fix it takes around 2 seconds now. 10x time improvement in line with pacer's sleep reduction from 100ms to 10ms --- backend/oracleobjectstorage/options.go | 4 +--- backend/oracleobjectstorage/oracleobjectstorage.go | 8 ++++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/oracleobjectstorage/options.go b/backend/oracleobjectstorage/options.go index 2916157be..004465093 100644 --- a/backend/oracleobjectstorage/options.go +++ b/backend/oracleobjectstorage/options.go @@ -17,9 +17,7 @@ const ( defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024) defaultUploadConcurrency = 10 maxUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024 * 1024) - minSleep = 100 * time.Millisecond - maxSleep = 5 * time.Minute - decayConstant = 1 // bigger for slower decay, exponential + minSleep = 10 * time.Millisecond defaultCopyTimeoutDuration = fs.Duration(time.Minute) ) diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.go index 6f730f258..c5f64744d 100644 --- a/backend/oracleobjectstorage/oracleobjectstorage.go +++ b/backend/oracleobjectstorage/oracleobjectstorage.go @@ -64,14 +64,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e if err != nil { return nil, err } - p := pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)) + pc := fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))) + // Set pacer retries to 2 (1 try and 1 retry) because we are + // relying on SDK retry mechanism, but we allow 2 attempts to + // retry directory listings after XMLSyntaxError + pc.SetRetries(2) f := &Fs{ name: name, opt: *opt, ci: ci, srv: objectStorageClient, cache: bucket.NewCache(), - pacer: fs.NewPacer(ctx, p), + pacer: pc, } f.setRoot(root) f.features = (&fs.Features{