fcb912a664
when multi-thread downloading is enabled, rclone used to send a write to disk after every read, resulting in a lot of small writes to different locations of the file. depending on the underlying filesystem or device, it can be more efficient to send bigger writes.
276 lines
9.3 KiB
Go
276 lines
9.3 KiB
Go
package fs
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Global
|
|
var (
|
|
// globalConfig for rclone
|
|
globalConfig = NewConfig()
|
|
|
|
// Read a value from the config file
|
|
//
|
|
// This is a function pointer to decouple the config
|
|
// implementation from the fs
|
|
ConfigFileGet = func(section, key string) (string, bool) { return "", false }
|
|
|
|
// Set a value into the config file and persist it
|
|
//
|
|
// This is a function pointer to decouple the config
|
|
// implementation from the fs
|
|
ConfigFileSet = func(section, key, value string) (err error) {
|
|
return errors.New("no config file set handler")
|
|
}
|
|
|
|
// Check if the config file has the named section
|
|
//
|
|
// This is a function pointer to decouple the config
|
|
// implementation from the fs
|
|
ConfigFileHasSection = func(section string) bool { return false }
|
|
|
|
// CountError counts an error. If any errors have been
|
|
// counted then rclone will exit with a non zero error code.
|
|
//
|
|
// This is a function pointer to decouple the config
|
|
// implementation from the fs
|
|
CountError = func(err error) error { return err }
|
|
|
|
// ConfigProvider is the config key used for provider options
|
|
ConfigProvider = "provider"
|
|
|
|
// ConfigEdit is the config key used to show we wish to edit existing entries
|
|
ConfigEdit = "config_fs_edit"
|
|
)
|
|
|
|
// ConfigInfo is filesystem config options
|
|
type ConfigInfo struct {
|
|
LogLevel LogLevel
|
|
StatsLogLevel LogLevel
|
|
UseJSONLog bool
|
|
DryRun bool
|
|
Interactive bool
|
|
CheckSum bool
|
|
SizeOnly bool
|
|
IgnoreTimes bool
|
|
IgnoreExisting bool
|
|
IgnoreErrors bool
|
|
ModifyWindow time.Duration
|
|
Checkers int
|
|
Transfers int
|
|
ConnectTimeout time.Duration // Connect timeout
|
|
Timeout time.Duration // Data channel timeout
|
|
ExpectContinueTimeout time.Duration
|
|
Dump DumpFlags
|
|
InsecureSkipVerify bool // Skip server certificate verification
|
|
DeleteMode DeleteMode
|
|
MaxDelete int64
|
|
MaxDeleteSize SizeSuffix
|
|
TrackRenames bool // Track file renames.
|
|
TrackRenamesStrategy string // Comma separated list of strategies used to track renames
|
|
LowLevelRetries int
|
|
UpdateOlder bool // Skip files that are newer on the destination
|
|
NoGzip bool // Disable compression
|
|
MaxDepth int
|
|
IgnoreSize bool
|
|
IgnoreChecksum bool
|
|
IgnoreCaseSync bool
|
|
NoTraverse bool
|
|
CheckFirst bool
|
|
NoCheckDest bool
|
|
NoUnicodeNormalization bool
|
|
NoUpdateModTime bool
|
|
DataRateUnit string
|
|
CompareDest []string
|
|
CopyDest []string
|
|
BackupDir string
|
|
Suffix string
|
|
SuffixKeepExtension bool
|
|
UseListR bool
|
|
BufferSize SizeSuffix
|
|
MultiThreadWriteBufferSize SizeSuffix
|
|
BwLimit BwTimetable
|
|
BwLimitFile BwTimetable
|
|
TPSLimit float64
|
|
TPSLimitBurst int
|
|
BindAddr net.IP
|
|
DisableFeatures []string
|
|
UserAgent string
|
|
Immutable bool
|
|
AutoConfirm bool
|
|
StreamingUploadCutoff SizeSuffix
|
|
StatsFileNameLength int
|
|
AskPassword bool
|
|
PasswordCommand SpaceSepList
|
|
UseServerModTime bool
|
|
MaxTransfer SizeSuffix
|
|
MaxDuration time.Duration
|
|
CutoffMode CutoffMode
|
|
MaxBacklog int
|
|
MaxStatsGroups int
|
|
StatsOneLine bool
|
|
StatsOneLineDate bool // If we want a date prefix at all
|
|
StatsOneLineDateFormat string // If we want to customize the prefix
|
|
ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred
|
|
Progress bool
|
|
ProgressTerminalTitle bool
|
|
Cookie bool
|
|
UseMmap bool
|
|
CaCert []string // Client Side CA
|
|
ClientCert string // Client Side Cert
|
|
ClientKey string // Client Side Key
|
|
MultiThreadCutoff SizeSuffix
|
|
MultiThreadStreams int
|
|
MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags)
|
|
OrderBy string // instructions on how to order the transfer
|
|
UploadHeaders []*HTTPOption
|
|
DownloadHeaders []*HTTPOption
|
|
Headers []*HTTPOption
|
|
MetadataSet Metadata // extra metadata to write when uploading
|
|
RefreshTimes bool
|
|
NoConsole bool
|
|
TrafficClass uint8
|
|
FsCacheExpireDuration time.Duration
|
|
FsCacheExpireInterval time.Duration
|
|
DisableHTTP2 bool
|
|
HumanReadable bool
|
|
KvLockTime time.Duration // maximum time to keep key-value database locked by process
|
|
DisableHTTPKeepAlives bool
|
|
Metadata bool
|
|
ServerSideAcrossConfigs bool
|
|
TerminalColorMode TerminalColorMode
|
|
DefaultTime Time // time that directories with no time should display
|
|
Inplace bool // Download directly to destination file instead of atomic download to temp/rename
|
|
}
|
|
|
|
// NewConfig creates a new config with everything set to the default
|
|
// value. These are the ultimate defaults and are overridden by the
|
|
// config module.
|
|
func NewConfig() *ConfigInfo {
|
|
c := new(ConfigInfo)
|
|
|
|
// Set any values which aren't the zero for the type
|
|
c.LogLevel = LogLevelNotice
|
|
c.StatsLogLevel = LogLevelInfo
|
|
c.ModifyWindow = time.Nanosecond
|
|
c.Checkers = 8
|
|
c.Transfers = 4
|
|
c.ConnectTimeout = 60 * time.Second
|
|
c.Timeout = 5 * 60 * time.Second
|
|
c.ExpectContinueTimeout = 1 * time.Second
|
|
c.DeleteMode = DeleteModeDefault
|
|
c.MaxDelete = -1
|
|
c.MaxDeleteSize = SizeSuffix(-1)
|
|
c.LowLevelRetries = 10
|
|
c.MaxDepth = -1
|
|
c.DataRateUnit = "bytes"
|
|
c.BufferSize = SizeSuffix(16 << 20)
|
|
c.MultiThreadWriteBufferSize = SizeSuffix(128 * 1024)
|
|
c.UserAgent = "rclone/" + Version
|
|
c.StreamingUploadCutoff = SizeSuffix(100 * 1024)
|
|
c.MaxStatsGroups = 1000
|
|
c.StatsFileNameLength = 45
|
|
c.AskPassword = true
|
|
c.TPSLimitBurst = 1
|
|
c.MaxTransfer = -1
|
|
c.MaxBacklog = 10000
|
|
// We do not want to set the default here. We use this variable being empty as part of the fall-through of options.
|
|
// c.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "
|
|
c.MultiThreadCutoff = SizeSuffix(250 * 1024 * 1024)
|
|
c.MultiThreadStreams = 4
|
|
|
|
c.TrackRenamesStrategy = "hash"
|
|
c.FsCacheExpireDuration = 300 * time.Second
|
|
c.FsCacheExpireInterval = 60 * time.Second
|
|
c.KvLockTime = 1 * time.Second
|
|
c.DefaultTime = Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
|
|
|
|
// Perform a simple check for debug flags to enable debug logging during the flag initialization
|
|
for argIndex, arg := range os.Args {
|
|
if strings.HasPrefix(arg, "-vv") && strings.TrimRight(arg, "v") == "-" {
|
|
c.LogLevel = LogLevelDebug
|
|
}
|
|
if arg == "--log-level=DEBUG" || (arg == "--log-level" && len(os.Args) > argIndex+1 && os.Args[argIndex+1] == "DEBUG") {
|
|
c.LogLevel = LogLevelDebug
|
|
}
|
|
if strings.HasPrefix(arg, "--verbose=") {
|
|
if level, err := strconv.Atoi(arg[10:]); err == nil && level >= 2 {
|
|
c.LogLevel = LogLevelDebug
|
|
}
|
|
}
|
|
}
|
|
envValue, found := os.LookupEnv("RCLONE_LOG_LEVEL")
|
|
if found && envValue == "DEBUG" {
|
|
c.LogLevel = LogLevelDebug
|
|
}
|
|
|
|
return c
|
|
}
|
|
|
|
// TimeoutOrInfinite returns ci.Timeout if > 0 or infinite otherwise
|
|
func (c *ConfigInfo) TimeoutOrInfinite() time.Duration {
|
|
if c.Timeout > 0 {
|
|
return c.Timeout
|
|
}
|
|
return ModTimeNotSupported
|
|
}
|
|
|
|
type configContextKeyType struct{}
|
|
|
|
// Context key for config
|
|
var configContextKey = configContextKeyType{}
|
|
|
|
// GetConfig returns the global or context sensitive context
|
|
func GetConfig(ctx context.Context) *ConfigInfo {
|
|
if ctx == nil {
|
|
return globalConfig
|
|
}
|
|
c := ctx.Value(configContextKey)
|
|
if c == nil {
|
|
return globalConfig
|
|
}
|
|
return c.(*ConfigInfo)
|
|
}
|
|
|
|
// CopyConfig copies the global config (if any) from srcCtx into
|
|
// dstCtx returning the new context.
|
|
func CopyConfig(dstCtx, srcCtx context.Context) context.Context {
|
|
if srcCtx == nil {
|
|
return dstCtx
|
|
}
|
|
c := srcCtx.Value(configContextKey)
|
|
if c == nil {
|
|
return dstCtx
|
|
}
|
|
return context.WithValue(dstCtx, configContextKey, c)
|
|
}
|
|
|
|
// AddConfig returns a mutable config structure based on a shallow
|
|
// copy of that found in ctx and returns a new context with that added
|
|
// to it.
|
|
func AddConfig(ctx context.Context) (context.Context, *ConfigInfo) {
|
|
c := GetConfig(ctx)
|
|
cCopy := new(ConfigInfo)
|
|
*cCopy = *c
|
|
newCtx := context.WithValue(ctx, configContextKey, cCopy)
|
|
return newCtx, cCopy
|
|
}
|
|
|
|
// ConfigToEnv converts a config section and name, e.g. ("my-remote",
|
|
// "ignore-size") into an environment name
|
|
// "RCLONE_CONFIG_MY-REMOTE_IGNORE_SIZE"
|
|
func ConfigToEnv(section, name string) string {
|
|
return "RCLONE_CONFIG_" + strings.ToUpper(section+"_"+strings.ReplaceAll(name, "-", "_"))
|
|
}
|
|
|
|
// OptionToEnv converts an option name, e.g. "ignore-size" into an
|
|
// environment name "RCLONE_IGNORE_SIZE"
|
|
func OptionToEnv(name string) string {
|
|
return "RCLONE_" + strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
|
|
}
|