rclone/cmd/test/memory/memory.go

// Package memory provides the memory test command.
package memory

import (
	"context"
	"runtime"
	"sync"

	"github.com/rclone/rclone/cmd"
	"github.com/rclone/rclone/cmd/test"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/operations"
	"github.com/spf13/cobra"
)

func init() {
	test.Command.AddCommand(commandDefinition)
}

var commandDefinition = &cobra.Command{
	Use:   "memory remote:path",
	Short: `Load all the objects at remote:path into memory and report memory stats.`,
	Annotations: map[string]string{
		"versionIntroduced": "v1.55",
	},
	Run: func(command *cobra.Command, args []string) {
		cmd.CheckArgs(1, 1, command, args)
		fsrc := cmd.NewFsSrc(args)
		cmd.Run(false, false, command, func() error {
			ctx := context.Background()
			ci := fs.GetConfig(context.Background())
			metadata := ci.Metadata && fsrc.Features().ReadMetadata
			objects, _, _, err := operations.Count(ctx, fsrc)
			if err != nil {
				return err
			}
			objs := make([]fs.Object, 0, objects)
			var before, after runtime.MemStats
			runtime.GC()
			runtime.ReadMemStats(&before)
			var mu sync.Mutex
			err = operations.ListFn(ctx, fsrc, func(o fs.Object) {
				// Read the metadata so it gets cached in the object
				if metadata {
					_, err := fs.GetMetadata(ctx, o)
					if err != nil {
						fs.Errorf(o, "Failed to read metadata: %v", err)
					}
				}
				mu.Lock()
				objs = append(objs, o)
				mu.Unlock()
			})
			if err != nil {
				return err
			}
			runtime.GC()
			runtime.ReadMemStats(&after)
			var allocChange int64
			if after.Alloc >= before.Alloc {
				allocChange = int64(after.Alloc - before.Alloc)
			} else {
				allocChange = -int64(before.Alloc - after.Alloc)
			}
			var sysChange int64
			if after.Sys >= before.Sys {
				sysChange = int64(after.Sys - before.Sys)
			} else {
				sysChange = -int64(before.Sys - after.Sys)
			}
			if ci.HumanReadable {
				objString := fs.CountSuffix(int64(len(objs)))
				var usedString string
				if after.Alloc >= before.Alloc {
					usedString = fs.SizeSuffix(int64(after.Alloc - before.Alloc)).ByteUnit()
				} else {
					usedString = "-" + fs.SizeSuffix(int64(before.Alloc-after.Alloc)).ByteUnit()
				}
				avgString := fs.SizeSuffix(allocChange / int64(len(objs))).ByteUnit()
				fs.Logf(nil, "%s objects took %s, %s/object", objString, usedString, avgString)

				var sysBeforeString string
				if before.Sys <= fs.SizeSuffixMaxValue {
					sysBeforeString = fs.SizeSuffix(int64(before.Sys)).String()
				} else {
					sysBeforeString = ">" + fs.SizeSuffixMax.String()
				}
				var sysAfterString string
				if after.Sys <= fs.SizeSuffixMaxValue {
					sysAfterString = fs.SizeSuffix(int64(after.Sys)).ByteUnit()
				} else {
					sysAfterString = ">" + fs.SizeSuffixMax.ByteUnit()
				}
				var sysUsedString string
				if after.Sys >= before.Sys {
					sysUsedString = fs.SizeSuffix(int64(after.Sys - before.Sys)).ByteUnit()
				} else {
					sysUsedString = "-" + fs.SizeSuffix(int64(before.Sys-after.Sys)).ByteUnit()
				}
				fs.Logf(nil, "System memory changed from %s to %s a change of %s", sysBeforeString, sysAfterString, sysUsedString)
			} else {
				fs.Logf(nil, "%d objects took %d bytes, %.1f bytes/object", len(objs), allocChange, float64(allocChange)/float64(len(objs)))
				fs.Logf(nil, "System memory changed from %d to %d bytes a change of %d bytes", before.Sys, after.Sys, sysChange)
			}
			return nil
		})
	},
}