log: add --use-json-log for JSON logging

docs/content/docs.md

@@ -635,6 +635,11 @@ warnings and significant events.
 
 `ERROR` is equivalent to `-q`. It only outputs error messages.
 
+### --use-json-log ###
+
+This switches the log format to JSON for rclone. The fields of json log 
+are level, msg, source, time.
+
 ### --low-level-retries NUMBER ###
 
 This controls the number of low level retries rclone does.

fs/config.go

@@ -40,6 +40,7 @@ var (
 type ConfigInfo struct {
 	LogLevel               LogLevel
 	StatsLogLevel          LogLevel
+	UseJSONLog             bool
 	DryRun                 bool
 	CheckSum               bool
 	SizeOnly               bool

fs/config/configflags/configflags.go

@@ -12,7 +12,9 @@ import (
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/flags"
+	fsLog "github.com/ncw/rclone/fs/log"
 	"github.com/ncw/rclone/fs/rc"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
 )
 
@@ -102,6 +104,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.IntVarP(flagSet, &fs.Config.MultiThreadStreams, "multi-thread-streams", "", fs.Config.MultiThreadStreams, "Max number of streams to use for multi-thread downloads.")
 	flags.DurationVarP(flagSet, &fs.Config.RcJobExpireDuration, "rc-job-expire-duration", "", fs.Config.RcJobExpireDuration, "expire finished async jobs older than this value")
 	flags.DurationVarP(flagSet, &fs.Config.RcJobExpireInterval, "rc-job-expire-interval", "", fs.Config.RcJobExpireInterval, "interval to check for expired async jobs")
+	flags.BoolVarP(flagSet, &fs.Config.UseJSONLog, "use-json-log", "", fs.Config.UseJSONLog, "Use json log format.")
 }
 
 // SetFlags converts any flags into config which weren't straight forward
@@ -126,6 +129,27 @@ func SetFlags() {
 			log.Fatalf("Can't set -q and --log-level")
 		}
 	}
+	if fs.Config.UseJSONLog {
+		logrus.AddHook(fsLog.NewCallerHook())
+		logrus.SetFormatter(&logrus.JSONFormatter{
+			TimestampFormat: "2006-01-02T15:04:05.999999-07:00",
+		})
+		logrus.SetLevel(logrus.DebugLevel)
+		switch fs.Config.LogLevel {
+		case fs.LogLevelEmergency, fs.LogLevelAlert:
+			logrus.SetLevel(logrus.PanicLevel)
+		case fs.LogLevelCritical:
+			logrus.SetLevel(logrus.FatalLevel)
+		case fs.LogLevelError:
+			logrus.SetLevel(logrus.ErrorLevel)
+		case fs.LogLevelWarning, fs.LogLevelNotice:
+			logrus.SetLevel(logrus.WarnLevel)
+		case fs.LogLevelInfo:
+			logrus.SetLevel(logrus.InfoLevel)
+		case fs.LogLevelDebug:
+			logrus.SetLevel(logrus.DebugLevel)
+		}
+	}
 
 	if dumpHeaders {
 		fs.Config.Dump |= fs.DumpHeaders

fs/log.go

@@ -5,6 +5,7 @@ import (
 	"log"
 
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // LogLevel describes rclone's logs.  These are a subset of the syslog log levels.
@@ -79,7 +80,24 @@ func LogPrintf(level LogLevel, o interface{}, text string, args ...interface{})
 	if o != nil {
 		out = fmt.Sprintf("%v: %s", o, out)
 	}
-	LogPrint(level, out)
+	if Config.UseJSONLog {
+		switch level {
+		case LogLevelDebug:
+			logrus.Debug(out)
+		case LogLevelInfo:
+			logrus.Info(out)
+		case LogLevelNotice, LogLevelWarning:
+			logrus.Warn(out)
+		case LogLevelError:
+			logrus.Error(out)
+		case LogLevelCritical:
+			logrus.Fatal(out)
+		case LogLevelEmergency, LogLevelAlert:
+			logrus.Panic(out)
+		}
+	} else {
+		LogPrint(level, out)
+	}
 }
 
 // LogLevelPrintf writes logs at the given level

fs/log/caller_hook.go

@@ -0,0 +1,72 @@
+package log
+
+import (
+	"fmt"
+	"runtime"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+)
+
+// CallerHook for log the calling file and line of the fine
+type CallerHook struct {
+	Field  string
+	Skip   int
+	levels []logrus.Level
+}
+
+// NewCallerHook use to make an hook
+func NewCallerHook(levels ...logrus.Level) logrus.Hook {
+	hook := CallerHook{
+		Field:  "source",
+		Skip:   7,
+		levels: levels,
+	}
+	if len(hook.levels) == 0 {
+		hook.levels = logrus.AllLevels
+	}
+	return &hook
+}
+
+// Levels implement applied hook to which levels
+func (h *CallerHook) Levels() []logrus.Level {
+	return logrus.AllLevels
+}
+
+// Fire logs the information of context (filename and line)
+func (h *CallerHook) Fire(entry *logrus.Entry) error {
+	entry.Data[h.Field] = findCaller(h.Skip)
+	return nil
+}
+
+// findCaller ignores the caller relevent to logrus or fslog then find out the exact caller
+func findCaller(skip int) string {
+	file := ""
+	line := 0
+	for i := 0; i < 10; i++ {
+		file, line = getCaller(skip + i)
+		if !strings.HasPrefix(file, "logrus") && strings.Index(file, "log.go") < 0 {
+			break
+		}
+	}
+	return fmt.Sprintf("%s:%d", file, line)
+}
+
+func getCaller(skip int) (string, int) {
+	_, file, line, ok := runtime.Caller(skip)
+	// fmt.Println(file,":",line)
+	if !ok {
+		return "", 0
+	}
+	n := 0
+	for i := len(file) - 1; i > 0; i-- {
+		if file[i] == '/' {
+			n++
+			if n >= 2 {
+				file = file[i+1:]
+				break
+			}
+		}
+	}
+	return file, line
+}