sync: don't need to sync directories if they haven't been modified

Before this change we synced directories regardless if the source
directory existed. It is irrelevant whether the source directory
exists or not, what we need to know is has the directory been
modified.

Co-authored-by: nielash <nielronash@gmail.com>

fs/sync/sync.go

@@ -1203,10 +1203,8 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {
 			if gCtx.Err() != nil {
 				break
 			}
-			if item.src == nil {
-				if _, ok := s.modifiedDirs[item.dir]; !ok {
-					continue
-				}
+			if _, ok := s.modifiedDirs[item.dir]; !ok {
+				continue
 			}
 			if !s.copyEmptySrcDirs {
 				if _, isEmpty := s.srcEmptyDirs[item.dir]; isEmpty {