rclone/fs/config.go
Ole Frost 58c99427b3 config: fixed issues with flags/options set by environment vars.
Some environment variables didn’t behave like their corresponding
command line flags. The affected flags were --stats, --log-level,
--separator, --multi-tread-streams, --rc-addr, --rc-user and --rc-pass.
Example:

    RCLONE_STATS='10s'
    rclone check remote: remote: --progress
    # Expected: rclone check remote: remote: --progress –-stats=10s
    # Actual: rclone check remote: remote: --progress

Remote specific options set by environment variables was overruled by
less specific backend options set by environment variables. Example:

    RCLONE_DRIVE_USE_TRASH='false'
    RCLONE_CONFIG_MYDRIVE_USE_TRASH='true'
    rclone deletefile myDrive:my-test-file
    # Expected: my-test-file is recoverable in the trash folder
    # Actual: my-test-file is permanently deleted (not recoverable)

Backend specific options set by environment variables was overruled by
general backend options set by environment variables. Example:

    RCLONE_SKIP_LINKS='true'
    RCLONE_LOCAL_SKIP_LINKS='false'
    rclone lsd local:
    # Expected result: Warnings when symlinks are skipped
    # Actual result: No warnings when symlinks are skipped
    # That is RCLONE_SKIP_LINKS takes precedence

The above issues have been fixed.

The debug logging (-vv) has been enhanced to show when flags are set by
environment variables.

The documentation has been enhanced with details on the precedence of
configuration options.

See pull request #5341 for more information.
2021-07-05 16:38:20 +01:00

250 lines
7.8 KiB
Go

package fs
import (
"context"
"net"
"os"
"strings"
"time"
"github.com/pkg/errors"
)
// Global
var (
// globalConfig for rclone
globalConfig = NewConfig()
// Read a value from the config file
//
// This is a function pointer to decouple the config
// implementation from the fs
ConfigFileGet = func(section, key string) (string, bool) { return "", false }
// Set a value into the config file and persist it
//
// This is a function pointer to decouple the config
// implementation from the fs
ConfigFileSet = func(section, key, value string) (err error) {
return errors.New("no config file set handler")
}
// CountError counts an error. If any errors have been
// counted then rclone will exit with a non zero error code.
//
// This is a function pointer to decouple the config
// implementation from the fs
CountError = func(err error) error { return err }
// ConfigProvider is the config key used for provider options
ConfigProvider = "provider"
// ConfigEdit is the config key used to show we wish to edit existing entries
ConfigEdit = "config_fs_edit"
)
// ConfigInfo is filesystem config options
type ConfigInfo struct {
LogLevel LogLevel
StatsLogLevel LogLevel
UseJSONLog bool
DryRun bool
Interactive bool
CheckSum bool
SizeOnly bool
IgnoreTimes bool
IgnoreExisting bool
IgnoreErrors bool
ModifyWindow time.Duration
Checkers int
Transfers int
ConnectTimeout time.Duration // Connect timeout
Timeout time.Duration // Data channel timeout
ExpectContinueTimeout time.Duration
Dump DumpFlags
InsecureSkipVerify bool // Skip server certificate verification
DeleteMode DeleteMode
MaxDelete int64
TrackRenames bool // Track file renames.
TrackRenamesStrategy string // Comma separated list of strategies used to track renames
LowLevelRetries int
UpdateOlder bool // Skip files that are newer on the destination
NoGzip bool // Disable compression
MaxDepth int
IgnoreSize bool
IgnoreChecksum bool
IgnoreCaseSync bool
NoTraverse bool
CheckFirst bool
NoCheckDest bool
NoUnicodeNormalization bool
NoUpdateModTime bool
DataRateUnit string
CompareDest []string
CopyDest []string
BackupDir string
Suffix string
SuffixKeepExtension bool
UseListR bool
BufferSize SizeSuffix
BwLimit BwTimetable
BwLimitFile BwTimetable
TPSLimit float64
TPSLimitBurst int
BindAddr net.IP
DisableFeatures []string
UserAgent string
Immutable bool
AutoConfirm bool
StreamingUploadCutoff SizeSuffix
StatsFileNameLength int
AskPassword bool
PasswordCommand SpaceSepList
UseServerModTime bool
MaxTransfer SizeSuffix
MaxDuration time.Duration
CutoffMode CutoffMode
MaxBacklog int
MaxStatsGroups int
StatsOneLine bool
StatsOneLineDate bool // If we want a date prefix at all
StatsOneLineDateFormat string // If we want to customize the prefix
ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred
Progress bool
ProgressTerminalTitle bool
Cookie bool
UseMmap bool
CaCert string // Client Side CA
ClientCert string // Client Side Cert
ClientKey string // Client Side Key
MultiThreadCutoff SizeSuffix
MultiThreadStreams int
MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags)
OrderBy string // instructions on how to order the transfer
UploadHeaders []*HTTPOption
DownloadHeaders []*HTTPOption
Headers []*HTTPOption
RefreshTimes bool
NoConsole bool
TrafficClass uint8
FsCacheExpireDuration time.Duration
FsCacheExpireInterval time.Duration
DisableHTTP2 bool
}
// NewConfig creates a new config with everything set to the default
// value. These are the ultimate defaults and are overridden by the
// config module.
func NewConfig() *ConfigInfo {
c := new(ConfigInfo)
// Set any values which aren't the zero for the type
c.LogLevel = LogLevelNotice
c.StatsLogLevel = LogLevelInfo
c.ModifyWindow = time.Nanosecond
c.Checkers = 8
c.Transfers = 4
c.ConnectTimeout = 60 * time.Second
c.Timeout = 5 * 60 * time.Second
c.ExpectContinueTimeout = 1 * time.Second
c.DeleteMode = DeleteModeDefault
c.MaxDelete = -1
c.LowLevelRetries = 10
c.MaxDepth = -1
c.DataRateUnit = "bytes"
c.BufferSize = SizeSuffix(16 << 20)
c.UserAgent = "rclone/" + Version
c.StreamingUploadCutoff = SizeSuffix(100 * 1024)
c.MaxStatsGroups = 1000
c.StatsFileNameLength = 45
c.AskPassword = true
c.TPSLimitBurst = 1
c.MaxTransfer = -1
c.MaxBacklog = 10000
// We do not want to set the default here. We use this variable being empty as part of the fall-through of options.
// c.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "
c.MultiThreadCutoff = SizeSuffix(250 * 1024 * 1024)
c.MultiThreadStreams = 4
c.TrackRenamesStrategy = "hash"
c.FsCacheExpireDuration = 300 * time.Second
c.FsCacheExpireInterval = 60 * time.Second
// Perform a simple check for debug flags to enable debug logging during the flag initialization
for argIndex, arg := range os.Args {
if strings.HasPrefix(arg, "-vv") && strings.TrimRight(arg, "v") == "-" {
c.LogLevel = LogLevelDebug
}
if arg == "--log-level=DEBUG" || (arg == "--log-level" && len(os.Args) > argIndex+1 && os.Args[argIndex+1] == "DEBUG") {
c.LogLevel = LogLevelDebug
}
}
envValue, found := os.LookupEnv("RCLONE_LOG_LEVEL")
if found && envValue == "DEBUG" {
c.LogLevel = LogLevelDebug
}
return c
}
// TimeoutOrInfinite returns ci.Timeout if > 0 or infinite otherwise
func (c *ConfigInfo) TimeoutOrInfinite() time.Duration {
if c.Timeout > 0 {
return c.Timeout
}
return ModTimeNotSupported
}
type configContextKeyType struct{}
// Context key for config
var configContextKey = configContextKeyType{}
// GetConfig returns the global or context sensitive context
func GetConfig(ctx context.Context) *ConfigInfo {
if ctx == nil {
return globalConfig
}
c := ctx.Value(configContextKey)
if c == nil {
return globalConfig
}
return c.(*ConfigInfo)
}
// CopyConfig copies the global config (if any) from srcCtx into
// dstCtx returning the new context.
func CopyConfig(dstCtx, srcCtx context.Context) context.Context {
if srcCtx == nil {
return dstCtx
}
c := srcCtx.Value(configContextKey)
if c == nil {
return dstCtx
}
return context.WithValue(dstCtx, configContextKey, c)
}
// AddConfig returns a mutable config structure based on a shallow
// copy of that found in ctx and returns a new context with that added
// to it.
func AddConfig(ctx context.Context) (context.Context, *ConfigInfo) {
c := GetConfig(ctx)
cCopy := new(ConfigInfo)
*cCopy = *c
newCtx := context.WithValue(ctx, configContextKey, cCopy)
return newCtx, cCopy
}
// ConfigToEnv converts a config section and name, e.g. ("myremote",
// "ignore-size") into an environment name
// "RCLONE_CONFIG_MYREMOTE_IGNORE_SIZE"
func ConfigToEnv(section, name string) string {
return "RCLONE_CONFIG_" + strings.ToUpper(strings.Replace(section+"_"+name, "-", "_", -1))
}
// OptionToEnv converts an option name, e.g. "ignore-size" into an
// environment name "RCLONE_IGNORE_SIZE"
func OptionToEnv(name string) string {
return "RCLONE_" + strings.ToUpper(strings.Replace(name, "-", "_", -1))
}