backup: Only return a warning for duplicate directory entries

The backup command failed if a directory contains duplicate entries.
Downgrade the severity of this problem from fatal error to a warning.
This allows users to still create a backup.

changelog/unreleased/issue-1866

@@ -0,0 +1,12 @@
+Enhancement: Improve handling of directories with duplicate entries
+
+If for some reason a directory contains a duplicate entry, this caused the
+backup command to fail with a `node "path/to/file" already present` or `nodes
+are not ordered got "path/to/file", last "path/to/file"` error.
+
+The error handling has been changed to only report a warning in this case. Make
+sure to check that the filesystem in question is not damaged!
+
+https://github.com/restic/restic/issues/1866
+https://github.com/restic/restic/issues/3937
+https://github.com/restic/restic/pull/3880

internal/archiver/tree_saver.go

@@ -2,6 +2,7 @@ package archiver
 
 import (
 	"context"
+	"errors"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/restic"
@@ -79,6 +80,7 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I
 	job.nodes = nil
 
 	builder := restic.NewTreeJSONBuilder()
+	var lastNode *restic.Node
 
 	for i, fn := range nodes {
 		// fn is a copy, so clear the original value explicitly
@@ -105,9 +107,15 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I
 
 		debug.Log("insert %v", fnr.node.Name)
 		err := builder.AddNode(fnr.node)
+		if err != nil && errors.Is(err, restic.ErrTreeNotOrdered) && lastNode != nil && fnr.node.Equals(*lastNode) {
+			// ignore error if an _identical_ node already exists, but nevertheless issue a warning
+			_ = s.errFn(fnr.target, err)
+			err = nil
+		}
 		if err != nil {
 			return nil, stats, err
 		}
+		lastNode = fnr.node
 	}
 
 	buf, err := builder.Finalize()

internal/restic/tree.go

@@ -145,6 +145,8 @@ func SaveTree(ctx context.Context, r BlobSaver, t *Tree) (ID, error) {
 	return id, err
 }
 
+var ErrTreeNotOrdered = errors.Errorf("nodes are not ordered or duplicate")
+
 type TreeJSONBuilder struct {
 	buf      bytes.Buffer
 	lastName string
@@ -158,7 +160,7 @@ func NewTreeJSONBuilder() *TreeJSONBuilder {
 
 func (builder *TreeJSONBuilder) AddNode(node *Node) error {
 	if node.Name <= builder.lastName {
-		return errors.Errorf("nodes are not ordered got %q, last %q", node.Name, builder.lastName)
+		return fmt.Errorf("node %q, last%q: %w", node.Name, builder.lastName, ErrTreeNotOrdered)
 	}
 	if builder.lastName != "" {
 		_ = builder.buf.WriteByte(',')

internal/restic/tree_test.go

@@ -3,6 +3,7 @@ package restic_test
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -136,6 +137,7 @@ func TestTreeEqualSerialization(t *testing.T) {
 
 			rtest.Assert(t, tree.Insert(node) != nil, "no error on duplicate node")
 			rtest.Assert(t, builder.AddNode(node) != nil, "no error on duplicate node")
+			rtest.Assert(t, errors.Is(builder.AddNode(node), restic.ErrTreeNotOrdered), "wrong error returned")
 		}
 
 		treeBytes, err := json.Marshal(tree)