diff --git a/backend/cache/cache.go b/backend/cache/cache.go
index 13d1143e9..f91fb3e3d 100644
--- a/backend/cache/cache.go
+++ b/backend/cache/cache.go
@@ -385,6 +385,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 		cleanupChan:      make(chan bool, 1),
 		notifiedRemotes:  make(map[string]bool),
 	}
+	cache.PinUntilFinalized(f.Fs, f)
 	f.rateLimiter = rate.NewLimiter(rate.Limit(float64(opt.Rps)), opt.TotalWorkers)
 
 	f.plexConnector = &plexConnector{}
diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.go
index 8032bb6e4..5bdfa7a4d 100644
--- a/backend/chunker/chunker.go
+++ b/backend/chunker/chunker.go
@@ -262,6 +262,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		root: rpath,
 		opt:  *opt,
 	}
+	cache.PinUntilFinalized(f.base, f)
 	f.dirSort = true // processEntries requires that meta Objects prerun data chunks atm.
 
 	if err := f.configure(opt.NameFormat, opt.MetaFormat, opt.HashType); err != nil {
diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go
index b8f27f840..0ba57d9e4 100644
--- a/backend/crypt/crypt.go
+++ b/backend/crypt/crypt.go
@@ -186,6 +186,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		opt:    *opt,
 		cipher: cipher,
 	}
+	cache.PinUntilFinalized(f.Fs, f)
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go
index d8679a091..49a993b28 100644
--- a/backend/union/upstream/upstream.go
+++ b/backend/union/upstream/upstream.go
@@ -97,6 +97,7 @@ func New(remote, root string, cacheTime time.Duration) (*Fs, error) {
 		return nil, err
 	}
 	f.Fs = myFs
+	cache.PinUntilFinalized(f.Fs, f)
 	return f, err
 }
 
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 1d606a4d4..8b06b096a 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -89,8 +89,10 @@ func NewFsFile(remote string) (fs.Fs, string) {
 	f, err := cache.Get(remote)
 	switch err {
 	case fs.ErrorIsFile:
+		cache.Pin(f) // pin indefinitely since it was on the CLI
 		return f, path.Base(fsPath)
 	case nil:
+		cache.Pin(f) // pin indefinitely since it was on the CLI
 		return f, ""
 	default:
 		err = fs.CountError(err)
@@ -139,6 +141,7 @@ func newFsDir(remote string) fs.Fs {
 		err = fs.CountError(err)
 		log.Fatalf("Failed to create file system for %q: %v", remote, err)
 	}
+	cache.Pin(f) // pin indefinitely since it was on the CLI
 	return f
 }
 
@@ -197,6 +200,7 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs
 		_ = fs.CountError(err)
 		log.Fatalf("Failed to create file system for destination %q: %v", dstRemote, err)
 	}
+	cache.Pin(fdst) // pin indefinitely since it was on the CLI
 	return
 }
 
diff --git a/fs/cache/cache.go b/fs/cache/cache.go
index 4a5bdc803..8257ea0dc 100644
--- a/fs/cache/cache.go
+++ b/fs/cache/cache.go
@@ -2,6 +2,7 @@
 package cache
 
 import (
+	"runtime"
 	"sync"
 
 	"github.com/rclone/rclone/fs"
@@ -80,6 +81,18 @@ func Pin(f fs.Fs) {
 	c.Pin(fs.ConfigString(f))
 }
 
+// PinUntilFinalized pins f into the cache until x is garbage collected
+//
+// This calls runtime.SetFinalizer on x so it shouldn't have a
+// finalizer already.
+func PinUntilFinalized(f fs.Fs, x interface{}) {
+	Pin(f)
+	runtime.SetFinalizer(x, func(_ interface{}) {
+		Unpin(f)
+	})
+
+}
+
 // Unpin f from the cache
 func Unpin(f fs.Fs) {
 	c.Pin(fs.ConfigString(f))
diff --git a/vfs/vfs.go b/vfs/vfs.go
index dd54bdb9a..d2d1d9d2b 100644
--- a/vfs/vfs.go
+++ b/vfs/vfs.go
@@ -234,8 +234,8 @@ func New(f fs.Fs, opt *vfscommon.Options) *VFS {
 
 	// Pin the Fs into the cache so that when we use cache.NewFs
 	// with the same remote string we get this one. The Pin is
-	// removed by Shutdown
-	cache.Pin(f)
+	// removed when the vfs is finalized
+	cache.PinUntilFinalized(f, vfs)
 
 	return vfs
 }
@@ -293,9 +293,6 @@ func (vfs *VFS) Shutdown() {
 		return
 	}
 
-	// Unpin the Fs from the cache
-	cache.Unpin(vfs.f)
-
 	// Remove from active cache
 	activeMu.Lock()
 	configName := fs.ConfigString(vfs.f)