108 lines
3.3 KiB
Go
108 lines
3.3 KiB
Go
// Package memory provides the memory test command.
|
|
package memory
|
|
|
|
import (
|
|
"context"
|
|
"runtime"
|
|
"sync"
|
|
|
|
"github.com/rclone/rclone/cmd"
|
|
"github.com/rclone/rclone/cmd/test"
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/operations"
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
func init() {
|
|
test.Command.AddCommand(commandDefinition)
|
|
}
|
|
|
|
var commandDefinition = &cobra.Command{
|
|
Use: "memory remote:path",
|
|
Short: `Load all the objects at remote:path into memory and report memory stats.`,
|
|
Annotations: map[string]string{
|
|
"versionIntroduced": "v1.55",
|
|
},
|
|
Run: func(command *cobra.Command, args []string) {
|
|
cmd.CheckArgs(1, 1, command, args)
|
|
fsrc := cmd.NewFsSrc(args)
|
|
cmd.Run(false, false, command, func() error {
|
|
ctx := context.Background()
|
|
ci := fs.GetConfig(context.Background())
|
|
metadata := ci.Metadata && fsrc.Features().ReadMetadata
|
|
objects, _, _, err := operations.Count(ctx, fsrc)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
objs := make([]fs.Object, 0, objects)
|
|
var before, after runtime.MemStats
|
|
runtime.GC()
|
|
runtime.ReadMemStats(&before)
|
|
var mu sync.Mutex
|
|
err = operations.ListFn(ctx, fsrc, func(o fs.Object) {
|
|
// Read the metadata so it gets cached in the object
|
|
if metadata {
|
|
_, err := fs.GetMetadata(ctx, o)
|
|
if err != nil {
|
|
fs.Errorf(o, "Failed to read metadata: %v", err)
|
|
}
|
|
}
|
|
mu.Lock()
|
|
objs = append(objs, o)
|
|
mu.Unlock()
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
runtime.GC()
|
|
runtime.ReadMemStats(&after)
|
|
var allocChange int64
|
|
if after.Alloc >= before.Alloc {
|
|
allocChange = int64(after.Alloc - before.Alloc)
|
|
} else {
|
|
allocChange = -int64(before.Alloc - after.Alloc)
|
|
}
|
|
var sysChange int64
|
|
if after.Sys >= before.Sys {
|
|
sysChange = int64(after.Sys - before.Sys)
|
|
} else {
|
|
sysChange = -int64(before.Sys - after.Sys)
|
|
}
|
|
if ci.HumanReadable {
|
|
objString := fs.CountSuffix(int64(len(objs)))
|
|
var usedString string
|
|
if after.Alloc >= before.Alloc {
|
|
usedString = fs.SizeSuffix(int64(after.Alloc - before.Alloc)).ByteUnit()
|
|
} else {
|
|
usedString = "-" + fs.SizeSuffix(int64(before.Alloc-after.Alloc)).ByteUnit()
|
|
}
|
|
avgString := fs.SizeSuffix(allocChange / int64(len(objs))).ByteUnit()
|
|
fs.Logf(nil, "%s objects took %s, %s/object", objString, usedString, avgString)
|
|
|
|
var sysBeforeString string
|
|
if before.Sys <= fs.SizeSuffixMaxValue {
|
|
sysBeforeString = fs.SizeSuffix(int64(before.Sys)).String()
|
|
} else {
|
|
sysBeforeString = ">" + fs.SizeSuffixMax.String()
|
|
}
|
|
var sysAfterString string
|
|
if after.Sys <= fs.SizeSuffixMaxValue {
|
|
sysAfterString = fs.SizeSuffix(int64(after.Sys)).ByteUnit()
|
|
} else {
|
|
sysAfterString = ">" + fs.SizeSuffixMax.ByteUnit()
|
|
}
|
|
var sysUsedString string
|
|
if after.Sys >= before.Sys {
|
|
sysUsedString = fs.SizeSuffix(int64(after.Sys - before.Sys)).ByteUnit()
|
|
} else {
|
|
sysUsedString = "-" + fs.SizeSuffix(int64(before.Sys-after.Sys)).ByteUnit()
|
|
}
|
|
fs.Logf(nil, "System memory changed from %s to %s a change of %s", sysBeforeString, sysAfterString, sysUsedString)
|
|
} else {
|
|
fs.Logf(nil, "%d objects took %d bytes, %.1f bytes/object", len(objs), allocChange, float64(allocChange)/float64(len(objs)))
|
|
fs.Logf(nil, "System memory changed from %d to %d bytes a change of %d bytes", before.Sys, after.Sys, sysChange)
|
|
}
|
|
return nil
|
|
})
|
|
},
|
|
}
|