size: warn about inaccurate results when objects with unknown size

cmd/size/size.go

@@ -32,18 +32,24 @@ var commandDefinition = &cobra.Command{
 			var results struct {
 				Count    int64 `json:"count"`
 				Bytes    int64 `json:"bytes"`
+				Sizeless int64 `json:"sizeless"`
 			}
 
-			results.Count, results.Bytes, err = operations.Count(context.Background(), fsrc)
+			results.Count, results.Bytes, results.Sizeless, err = operations.Count(context.Background(), fsrc)
 			if err != nil {
 				return err
 			}
-
+			if results.Sizeless > 0 {
+				fs.Logf(fsrc, "Size may be underestimated due to %d objects with unknown size", results.Sizeless)
+			}
 			if jsonOutput {
 				return json.NewEncoder(os.Stdout).Encode(results)
 			}
 			fmt.Printf("Total objects: %s (%d)\n", fs.CountSuffix(results.Count), results.Count)
 			fmt.Printf("Total size: %s (%d Byte)\n", fs.SizeSuffix(results.Bytes).ByteUnit(), results.Bytes)
+			if results.Sizeless > 0 {
+				fmt.Printf("Total objects with unknown size: %s (%d)\n", fs.CountSuffix(results.Sizeless), results.Sizeless)
+			}
 			return nil
 		})
 	},

cmd/test/memory/memory.go

@@ -25,7 +25,7 @@ var commandDefinition = &cobra.Command{
 		cmd.Run(false, false, command, func() error {
 			ctx := context.Background()
 			ci := fs.GetConfig(context.Background())
-			objects, _, err := operations.Count(ctx, fsrc)
+			objects, _, _, err := operations.Count(ctx, fsrc)
 			if err != nil {
 				return err
 			}

fs/operations/dedupe_test.go

@@ -121,7 +121,7 @@ func TestDeduplicateFirst(t *testing.T) {
 	// list until we get one object
 	var objects, size int64
 	for try := 1; try <= *fstest.ListRetries; try++ {
-		objects, size, err = operations.Count(context.Background(), r.Fremote)
+		objects, size, _, err = operations.Count(context.Background(), r.Fremote)
 		require.NoError(t, err)
 		if objects == 1 {
 			break

fs/operations/operations.go

@@ -1070,11 +1070,13 @@ func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Write
 // Count counts the objects and their sizes in the Fs
 //
 // Obeys includes and excludes
-func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, err error) {
+func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, sizelessObjects int64, err error) {
 	err = ListFn(ctx, f, func(o fs.Object) {
 		atomic.AddInt64(&objects, 1)
 		objectSize := o.Size()
-		if objectSize > 0 {
+		if objectSize < 0 {
+			atomic.AddInt64(&sizelessObjects, 1)
+		} else if objectSize > 0 {
 			atomic.AddInt64(&size, objectSize)
 		}
 	})

fs/operations/operations_test.go

@@ -401,10 +401,11 @@ func TestCount(t *testing.T) {
 	// Check the MaxDepth too
 	ci.MaxDepth = 1
 
-	objects, size, err := operations.Count(ctx, r.Fremote)
+	objects, size, sizeless, err := operations.Count(ctx, r.Fremote)
 	require.NoError(t, err)
 	assert.Equal(t, int64(2), objects)
 	assert.Equal(t, int64(61), size)
+	assert.Equal(t, int64(0), sizeless)
 }
 
 func TestDelete(t *testing.T) {

fs/operations/rc.go

@@ -344,13 +344,14 @@ func rcSize(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	if err != nil {
 		return nil, err
 	}
-	count, bytes, err := Count(ctx, f)
+	count, bytes, sizeless, err := Count(ctx, f)
 	if err != nil {
 		return nil, err
 	}
 	out = make(rc.Params)
 	out["count"] = count
 	out["bytes"] = bytes
+	out["sizeless"] = sizeless
 	return out, nil
 }
 

fs/operations/rc_test.go

@@ -453,6 +453,7 @@ func TestRcSize(t *testing.T) {
 	assert.Equal(t, rc.Params{
 		"count":    int64(3),
 		"bytes":    int64(120),
+		"sizeless": int64(0),
 	}, out)
 }