diff --git a/docs/content/rc.md b/docs/content/rc.md index 1869abb3d..40e7c0c46 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -85,6 +85,14 @@ style. Default Off. +### --rc-job-expire-duration=DURATION + +Expire finished async jobs older than DURATION (default 60s). + +### --rc-job-expire-interval=DURATION + +Interval duration to check for expired async jobs (default 10s). + ### --rc-no-auth By default rclone will require authorisation to have been set up on diff --git a/fs/config.go b/fs/config.go index e3700517b..6b77fc700 100644 --- a/fs/config.go +++ b/fs/config.go @@ -96,6 +96,8 @@ type ConfigInfo struct { ClientKey string // Client Side Key MultiThreadCutoff SizeSuffix MultiThreadStreams int + RcJobExpireDuration time.Duration + RcJobExpireInterval time.Duration } // NewConfig creates a new config with everything set to the default @@ -129,6 +131,8 @@ func NewConfig() *ConfigInfo { // c.StatsOneLineDateFormat = "2006/01/02 15:04:05 - " c.MultiThreadCutoff = SizeSuffix(250 * 1024 * 1024) c.MultiThreadStreams = 4 + c.RcJobExpireDuration = 60 * time.Second + c.RcJobExpireInterval = 10 * time.Second return c } diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index ef8de0493..f585e283d 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -98,6 +98,8 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.StringVarP(flagSet, &fs.Config.ClientKey, "client-key", "", fs.Config.ClientKey, "Client SSL private key (PEM) for mutual TLS auth") flags.FVarP(flagSet, &fs.Config.MultiThreadCutoff, "multi-thread-cutoff", "", "Use multi-thread downloads for files above this size.") flags.IntVarP(flagSet, &fs.Config.MultiThreadStreams, "multi-thread-streams", "", fs.Config.MultiThreadStreams, "Max number of streams to use for multi-thread downloads.") + flags.DurationVarP(flagSet, &fs.Config.RcJobExpireDuration, "rc-job-expire-duration", "", fs.Config.RcJobExpireDuration, "expire finished async jobs older than this value") + flags.DurationVarP(flagSet, &fs.Config.RcJobExpireInterval, "rc-job-expire-interval", "", fs.Config.RcJobExpireInterval, "interval to check for expired async jobs") } // SetFlags converts any flags into config which weren't straight forward diff --git a/fs/rc/job.go b/fs/rc/job.go index ea2621e0b..580a67c2d 100644 --- a/fs/rc/job.go +++ b/fs/rc/job.go @@ -7,16 +7,10 @@ import ( "sync/atomic" "time" + "github.com/ncw/rclone/fs" "github.com/pkg/errors" ) -const ( - // expire the job when it is finished and older than this - expireDuration = 60 * time.Second - // inteval to run the expire cache - expireInterval = 10 * time.Second -) - // Job describes a asynchronous task started via the rc package type Job struct { mu sync.Mutex @@ -47,7 +41,7 @@ var ( func newJobs() *Jobs { return &Jobs{ jobs: map[int64]*Job{}, - expireInterval: expireInterval, + expireInterval: fs.Config.RcJobExpireInterval, } } @@ -68,7 +62,7 @@ func (jobs *Jobs) Expire() { now := time.Now() for ID, job := range jobs.jobs { job.mu.Lock() - if job.Finished && now.Sub(job.EndTime) > expireDuration { + if job.Finished && now.Sub(job.EndTime) > fs.Config.RcJobExpireDuration { delete(jobs.jobs, ID) } job.mu.Unlock() diff --git a/fs/rc/job_test.go b/fs/rc/job_test.go index 97de9cc6f..805f6dee4 100644 --- a/fs/rc/job_test.go +++ b/fs/rc/job_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/ncw/rclone/fs" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -44,7 +45,7 @@ func TestJobsExpire(t *testing.T) { assert.Equal(t, 1, len(jobs.jobs)) jobs.mu.Lock() job.mu.Lock() - job.EndTime = time.Now().Add(-expireDuration - 60*time.Second) + job.EndTime = time.Now().Add(-fs.Config.RcJobExpireDuration - 60*time.Second) assert.Equal(t, true, jobs.expireRunning) job.mu.Unlock() jobs.mu.Unlock()