From ab040d8811ed48471ccfd8af21333cc1c80d8eb6 Mon Sep 17 00:00:00 2001 From: Igor Fedorenko Date: Sun, 11 Feb 2018 22:41:59 -0500 Subject: [PATCH] Introduced repository.DownloadAndHash helper Signed-off-by: Igor Fedorenko --- internal/checker/checker.go | 26 ++-------------------- internal/repository/repack.go | 25 +-------------------- internal/repository/repository.go | 37 +++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 48 deletions(-) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 5d75bcb2f..6e24b7b1c 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -2,15 +2,12 @@ package checker import ( "context" - "crypto/sha256" "fmt" "io" "os" "sync" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" @@ -630,9 +627,9 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID) error { debug.Log("checking pack %v", id) h := restic.Handle{Type: restic.DataFile, Name: id.String()} - packfile, err := fs.TempFile("", "restic-temp-check-") + packfile, hash, size, err := repository.DownloadAndHash(ctx, r, h) if err != nil { - return errors.Wrap(err, "TempFile") + return errors.Wrap(err, "checkPack") } defer func() { @@ -640,25 +637,6 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID) error { _ = os.Remove(packfile.Name()) }() - var hash restic.ID - var size int64 - err = r.Backend().Load(ctx, h, 0, 0, func(rd io.Reader) (ierr error) { - _, ierr = packfile.Seek(0, io.SeekStart) - if ierr == nil { - ierr = packfile.Truncate(0) - } - if ierr != nil { - return ierr - } - hrd := hashing.NewReader(rd, sha256.New()) - size, ierr = io.Copy(packfile, hrd) - hash = restic.IDFromHash(hrd.Sum(nil)) - return ierr - }) - if err != nil { - return errors.Wrap(err, "checkPack") - } - debug.Log("hash for pack %v is %v", id, hash) if !hash.Equal(id) { diff --git a/internal/repository/repack.go b/internal/repository/repack.go index 4bcd232d1..8f4c2caea 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -2,14 +2,11 @@ package repository import ( "context" - "crypto/sha256" "fmt" - "io" "os" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/restic" @@ -27,27 +24,7 @@ func Repack(ctx context.Context, repo restic.Repository, packs restic.IDSet, kee // load the complete pack into a temp file h := restic.Handle{Type: restic.DataFile, Name: packID.String()} - tempfile, err := fs.TempFile("", "restic-temp-repack-") - if err != nil { - return nil, errors.Wrap(err, "TempFile") - } - - // TODO very similar code in checker, consider moving to utils.go - var hash restic.ID - var packLength int64 - err = repo.Backend().Load(ctx, h, 0, 0, func(rd io.Reader) (ierr error) { - _, ierr = tempfile.Seek(0, io.SeekStart) - if ierr == nil { - ierr = tempfile.Truncate(0) - } - if ierr != nil { - return ierr - } - hrd := hashing.NewReader(rd, sha256.New()) - packLength, ierr = io.Copy(tempfile, hrd) - hash = restic.IDFromHash(hrd.Sum(nil)) - return ierr - }) + tempfile, hash, packLength, err := DownloadAndHash(ctx, repo, h) if err != nil { return nil, errors.Wrap(err, "Repack") } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 91af3085d..f22471ab5 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -3,12 +3,16 @@ package repository import ( "bytes" "context" + "crypto/sha256" "encoding/json" "fmt" + "io" "os" "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/backend" @@ -654,3 +658,36 @@ func (r *Repository) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, e _, err = r.SaveBlob(ctx, restic.TreeBlob, buf, id) return id, err } + +// DownloadAndHash is all-in-one helper to download content of the file at h to a temporary filesystem location +// and calculate ID of the contents. Returned (temporary) file is positioned at the beginning of the file; +// it is reponsibility of the caller to close and delete the file. +func DownloadAndHash(ctx context.Context, repo restic.Repository, h restic.Handle) (tmpfile *os.File, hash restic.ID, size int64, err error) { + tmpfile, err = fs.TempFile("", "restic-temp-") + if err != nil { + return nil, restic.ID{}, -1, errors.Wrap(err, "TempFile") + } + + err = repo.Backend().Load(ctx, h, 0, 0, func(rd io.Reader) (ierr error) { + _, ierr = tmpfile.Seek(0, io.SeekStart) + if ierr == nil { + ierr = tmpfile.Truncate(0) + } + if ierr != nil { + return ierr + } + hrd := hashing.NewReader(rd, sha256.New()) + size, ierr = io.Copy(tmpfile, hrd) + hash = restic.IDFromHash(hrd.Sum(nil)) + return ierr + }) + + _, err = tmpfile.Seek(0, io.SeekStart) + if err != nil { + tmpfile.Close() + os.Remove(tmpfile.Name()) + return nil, restic.ID{}, -1, errors.Wrap(err, "Seek") + } + + return tmpfile, hash, size, err +}