Merge pull request #2709 from greatroar/minio-sha256

Use Minio's optimized SHA-256

cmd/restic/cmd_stats.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -11,6 +10,8 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/walker"
+
+	"github.com/minio/sha256-simd"
 	"github.com/spf13/cobra"
 )
 

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/kurin/blazer v0.5.3
 	github.com/marstr/guid v1.1.0 // indirect
 	github.com/minio/minio-go/v6 v6.0.43
+	github.com/minio/sha256-simd v0.1.1
 	github.com/ncw/swift v1.0.47
 	github.com/pkg/errors v0.8.1
 	github.com/pkg/profile v1.3.0

internal/repository/packer_manager.go

@@ -2,7 +2,6 @@ package repository
 
 import (
 	"context"
-	"crypto/sha256"
 	"os"
 	"sync"
 
@@ -14,6 +13,8 @@ import (
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/pack"
+
+	"github.com/minio/sha256-simd"
 )
 
 // Saver implements saving data in a backend.

internal/repository/repository.go

@@ -3,7 +3,6 @@ package repository
 import (
 	"bytes"
 	"context"
-	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -17,6 +16,8 @@ import (
 	"github.com/restic/restic/internal/hashing"
 	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/restic"
+
+	"github.com/minio/sha256-simd"
 	"golang.org/x/sync/errgroup"
 )
 

internal/repository/repository_test.go

@@ -197,6 +197,10 @@ func BenchmarkLoadBlob(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		var err error
 		buf, err = repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
+
+		// Checking the SHA-256 with restic.Hash can make up 38% of the time
+		// spent in this loop, so pause the timer.
+		b.StopTimer()
 		rtest.OK(b, err)
 		if len(buf) != length {
 			b.Errorf("wanted %d bytes, got %d", length, len(buf))
@@ -206,6 +210,7 @@ func BenchmarkLoadBlob(b *testing.B) {
 		if !id.Equal(id2) {
 			b.Errorf("wrong data returned, wanted %v, got %v", id.Str(), id2.Str())
 		}
+		b.StartTimer()
 	}
 }
 
@@ -230,6 +235,9 @@ func BenchmarkLoadAndDecrypt(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		data, err := repo.LoadAndDecrypt(context.TODO(), nil, restic.DataFile, storageID)
 		rtest.OK(b, err)
+
+		// See comment in BenchmarkLoadBlob.
+		b.StopTimer()
 		if len(data) != length {
 			b.Errorf("wanted %d bytes, got %d", length, len(data))
 		}
@@ -238,6 +246,7 @@ func BenchmarkLoadAndDecrypt(b *testing.B) {
 		if !dataID.Equal(id2) {
 			b.Errorf("wrong data returned, wanted %v, got %v", storageID.Str(), id2.Str())
 		}
+		b.StartTimer()
 	}
 }
 

internal/restic/id.go

@@ -2,13 +2,14 @@ package restic
 
 import (
 	"crypto/rand"
-	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
 
 	"github.com/restic/restic/internal/errors"
+
+	"github.com/minio/sha256-simd"
 )
 
 // Hash returns the ID for data.