azureblob: ignore directory markers - fixes #2806
This ignores 0 length blobs if - they end with / - they have the metadata hdi_isfolder = true
This commit is contained in:
parent
d3e8ae1820
commit
feea0532cd
1 changed files with 26 additions and 5 deletions
|
@ -509,6 +509,7 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
|
||||||
MaxResults: int32(maxResults),
|
MaxResults: int32(maxResults),
|
||||||
}
|
}
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
directoryMarkers := map[string]struct{}{}
|
||||||
for marker := (azblob.Marker{}); marker.NotDone(); {
|
for marker := (azblob.Marker{}); marker.NotDone(); {
|
||||||
var response *azblob.ListBlobsHierarchySegmentResponse
|
var response *azblob.ListBlobsHierarchySegmentResponse
|
||||||
err := f.pacer.Call(func() (bool, error) {
|
err := f.pacer.Call(func() (bool, error) {
|
||||||
|
@ -538,13 +539,29 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
remote := file.Name[len(f.root):]
|
remote := file.Name[len(f.root):]
|
||||||
// Check for directory
|
// is this a directory marker?
|
||||||
isDirectory := strings.HasSuffix(remote, "/")
|
if *file.Properties.ContentLength == 0 {
|
||||||
if isDirectory {
|
// Note that metadata with hdi_isfolder = true seems to be a
|
||||||
remote = remote[:len(remote)-1]
|
// defacto standard for marking blobs as directories.
|
||||||
|
endsWithSlash := strings.HasSuffix(remote, "/")
|
||||||
|
if endsWithSlash || remote == "" || file.Metadata["hdi_isfolder"] == "true" {
|
||||||
|
if endsWithSlash {
|
||||||
|
remote = remote[:len(remote)-1]
|
||||||
|
}
|
||||||
|
err = fn(remote, file, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Keep track of directory markers. If recursing then
|
||||||
|
// there will be no Prefixes so no need to keep track
|
||||||
|
if !recurse {
|
||||||
|
directoryMarkers[remote] = struct{}{}
|
||||||
|
}
|
||||||
|
continue // skip directory marker
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Send object
|
// Send object
|
||||||
err = fn(remote, file, isDirectory)
|
err = fn(remote, file, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -557,6 +574,10 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
remote = remote[len(f.root):]
|
remote = remote[len(f.root):]
|
||||||
|
// Don't send if already sent as a directory marker
|
||||||
|
if _, found := directoryMarkers[remote]; found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
// Send object
|
// Send object
|
||||||
err = fn(remote, nil, true)
|
err = fn(remote, nil, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
Loading…
Reference in a new issue