KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #3831 from MichaelEischer/move-code

Browse source 
Move code out of the restic package and consolidate backend specific code
This commit is contained in:
MichaelEischer 2022-07-23 10:33:05 +02:00 committed by GitHub
commit 1f5369e072
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 53 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmd/restic/cmd_debug.go
View file

@ -107,7 +107,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), size)
blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), size)
if err != nil {
Warnf("error for pack %v: %v\n", id.Str(), err)
return nil
@ -525,7 +525,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
Printf(" ========================================\n")
Printf(" inspect the pack itself\n")
blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size)
blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), fi.Size)
if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err)
}

2
cmd/restic/global.go
View file

@ -16,6 +16,7 @@ import (
"github.com/restic/restic/internal/backend/azure"
"github.com/restic/restic/internal/backend/b2"
"github.com/restic/restic/internal/backend/gs"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/rclone"
@ -26,7 +27,6 @@ import (
"github.com/restic/restic/internal/cache"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/limiter"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"

3
internal/archiver/testing.go
View file

@ -11,6 +11,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
@ -225,7 +226,7 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Reposi
return
}
content := make([]byte, restic.CiphertextLength(len(file.Content)))
content := make([]byte, crypto.CiphertextLength(len(file.Content)))
pos := 0
for _, id := range node.Content {
part, err := repo.LoadBlob(ctx, restic.DataBlob, id, content[pos:])

2
internal/backend/backend_retry_test.go
View file

@ -7,8 +7,8 @@ import (
"io/ioutil"
"testing"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/mock"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)

0
internal/limiter/limiter.go → internal/backend/limiter/limiter.go
View file

0
internal/limiter/limiter_backend.go → internal/backend/limiter/limiter_backend.go
View file

2
internal/limiter/limiter_backend_test.go → internal/backend/limiter/limiter_backend_test.go
View file

@ -8,7 +8,7 @@ import (
"io"
"testing"
"github.com/restic/restic/internal/mock"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)

0
internal/limiter/static_limiter.go → internal/backend/limiter/static_limiter.go
View file

0
internal/limiter/static_limiter_test.go → internal/backend/limiter/static_limiter_test.go
View file

0
internal/mock/backend.go → internal/backend/mock/backend.go
View file

2
internal/backend/rclone/backend.go
View file

@ -16,10 +16,10 @@ import (
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/limiter"
"golang.org/x/net/context/ctxhttp"
"golang.org/x/net/http2"
)

11
internal/restic/readerat.go → internal/backend/readerat.go
View file

@ -1,4 +1,4 @@
package restic
package backend
import (
"context"
@ -6,12 +6,13 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
type backendReaderAt struct {
ctx context.Context
be Backend
h Handle
be restic.Backend
h restic.Handle
}
func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
@ -21,12 +22,12 @@ func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
// ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader
// should not escape the caller function to avoid unexpected interactions with the
// embedded context
func ReaderAt(ctx context.Context, be Backend, h Handle) io.ReaderAt {
func ReaderAt(ctx context.Context, be restic.Backend, h restic.Handle) io.ReaderAt {
return backendReaderAt{ctx: ctx, be: be, h: h}
}
// ReadAt reads from the backend handle h at the given position.
func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (n int, err error) {
func ReadAt(ctx context.Context, be restic.Backend, h restic.Handle, offset int64, p []byte) (n int, err error) {
debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p))
err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) {

2
internal/backend/utils_test.go
View file

@ -10,8 +10,8 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/mock"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)

5
internal/cache/file.go vendored
View file

@ -8,6 +8,7 @@ import (
"runtime"
"github.com/pkg/errors"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
@ -58,7 +59,7 @@ func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser,
return nil, errors.WithStack(err)
}
if fi.Size() <= int64(restic.CiphertextLength(0)) {
if fi.Size() <= int64(crypto.CiphertextLength(0)) {
_ = f.Close()
_ = c.remove(h)
return nil, errors.Errorf("cached file %v is truncated, removing", h)
@ -116,7 +117,7 @@ func (c *Cache) Save(h restic.Handle, rd io.Reader) error {
return errors.Wrap(err, "Copy")
}
if n <= int64(restic.CiphertextLength(0)) {
if n <= int64(crypto.CiphertextLength(0)) {
_ = f.Close()
_ = fs.Remove(f.Name())
debug.Log("trying to cache truncated file %v, removing", h)

10
internal/restic/buffer.go → internal/crypto/buffer.go
View file

@ -1,21 +1,19 @@
package restic
import "github.com/restic/restic/internal/crypto"
package crypto
// NewBlobBuffer returns a buffer that is large enough to hold a blob of size
// plaintext bytes, including the crypto overhead.
func NewBlobBuffer(size int) []byte {
return make([]byte, size, size+crypto.Extension)
return make([]byte, size, size+Extension)
}
// PlaintextLength returns the plaintext length of a blob with ciphertextSize
// bytes.
func PlaintextLength(ciphertextSize int) int {
return ciphertextSize - crypto.Extension
return ciphertextSize - Extension
}
// CiphertextLength returns the encrypted length of a blob with plaintextSize
// bytes.
func CiphertextLength(plaintextSize int) int {
return plaintextSize + crypto.Extension
return plaintextSize + Extension
}

6
internal/pack/pack.go
View file

@ -79,7 +79,7 @@ func (p *Packer) Finalize() error {
return err
}
encryptedHeader := make([]byte, 0, restic.CiphertextLength(len(header)))
encryptedHeader := make([]byte, 0, crypto.CiphertextLength(len(header)))
nonce := crypto.NewRandomNonce()
encryptedHeader = append(encryptedHeader, nonce...)
encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil)
@ -107,7 +107,7 @@ func (p *Packer) Finalize() error {
// HeaderOverhead returns an estimate of the number of bytes written by a call to Finalize.
func (p *Packer) HeaderOverhead() int {
return restic.CiphertextLength(0) + binary.Size(uint32(0))
return crypto.CiphertextLength(0) + binary.Size(uint32(0))
}
// makeHeader constructs the header for p.
@ -275,7 +275,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, hdr
return nil, 0, err
}
if len(buf) < restic.CiphertextLength(0) {
if len(buf) < crypto.CiphertextLength(0) {
return nil, 0, errors.New("invalid header, too small")
}

5
internal/pack/pack_test.go
View file

@ -9,6 +9,7 @@ import (
"io"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/pack"
@ -128,7 +129,7 @@ func TestUnpackReadSeeker(t *testing.T) {
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher())))
verifyBlobs(t, bufs, k, restic.ReaderAt(context.TODO(), b, handle), packSize)
verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize)
}
func TestShortPack(t *testing.T) {
@ -141,5 +142,5 @@ func TestShortPack(t *testing.T) {
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher())))
verifyBlobs(t, bufs, k, restic.ReaderAt(context.TODO(), b, handle), packSize)
verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize)
}

3
internal/repository/index.go
View file

@ -7,6 +7,7 @@ import (
"sync"
"time"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -194,7 +195,7 @@ func (idx *Index) LookupSize(bh restic.BlobHandle) (plaintextLength uint, found
if e.uncompressedLength != 0 {
return uint(e.uncompressedLength), true
}
return uint(restic.PlaintextLength(int(e.length))), true
return uint(crypto.PlaintextLength(int(e.length))), true
}
// Supersedes returns the list of indexes this index supersedes, if any.

2
internal/repository/key.go
View file

@ -263,7 +263,7 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str
}
nonce := crypto.NewRandomNonce()
ciphertext := make([]byte, 0, restic.CiphertextLength(len(buf)))
ciphertext := make([]byte, 0, crypto.CiphertextLength(len(buf)))
ciphertext = append(ciphertext, nonce...)
ciphertext = newkey.user.Seal(ciphertext, nonce, buf, nil)
newkey.Data = ciphertext

9
internal/repository/master_index_test.go
View file

@ -8,6 +8,7 @@ import (
"time"
"github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
@ -22,7 +23,7 @@ func TestMasterIndex(t *testing.T) {
PackID: restic.NewRandomID(),
Blob: restic.Blob{
BlobHandle: bhInIdx1,
Length: uint(restic.CiphertextLength(10)),
Length: uint(crypto.CiphertextLength(10)),
Offset: 0,
},
}
@ -31,7 +32,7 @@ func TestMasterIndex(t *testing.T) {
PackID: restic.NewRandomID(),
Blob: restic.Blob{
BlobHandle: bhInIdx2,
Length: uint(restic.CiphertextLength(100)),
Length: uint(crypto.CiphertextLength(100)),
Offset: 10,
UncompressedLength: 200,
},
@ -41,7 +42,7 @@ func TestMasterIndex(t *testing.T) {
PackID: restic.NewRandomID(),
Blob: restic.Blob{
BlobHandle: bhInIdx12,
Length: uint(restic.CiphertextLength(123)),
Length: uint(crypto.CiphertextLength(123)),
Offset: 110,
},
}
@ -50,7 +51,7 @@ func TestMasterIndex(t *testing.T) {
PackID: restic.NewRandomID(),
Blob: restic.Blob{
BlobHandle: bhInIdx12,
Length: uint(restic.CiphertextLength(123)),
Length: uint(crypto.CiphertextLength(123)),
Offset: 50,
UncompressedLength: 80,
},

9
internal/repository/repository.go
View file

@ -14,6 +14,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/klauspost/compress/zstd"
"github.com/restic/chunker"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/dryrun"
"github.com/restic/restic/internal/cache"
"github.com/restic/restic/internal/crypto"
@ -263,7 +264,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.
buf = buf[:blob.Length]
}
n, err := restic.ReadAt(ctx, r.be, h, int64(blob.Offset), buf)
n, err := backend.ReadAt(ctx, r.be, h, int64(blob.Offset), buf)
if err != nil {
debug.Log("error loading blob %v: %v", blob, err)
lastError = err
@ -398,7 +399,7 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
nonce := crypto.NewRandomNonce()
ciphertext := make([]byte, 0, restic.CiphertextLength(len(data)))
ciphertext := make([]byte, 0, crypto.CiphertextLength(len(data)))
ciphertext = append(ciphertext, nonce...)
// encrypt blob
@ -475,7 +476,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
}
}
ciphertext := restic.NewBlobBuffer(len(p))
ciphertext := crypto.NewBlobBuffer(len(p))
ciphertext = ciphertext[:0]
nonce := crypto.NewRandomNonce()
ciphertext = append(ciphertext, nonce...)
@ -793,7 +794,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) {
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
return pack.List(r.Key(), restic.ReaderAt(ctx, r.Backend(), h), size)
return pack.List(r.Key(), backend.ReaderAt(ctx, r.Backend(), h), size)
}
// Delete calls backend.Delete() if implemented, and returns an error

8
internal/repository/repository_test.go
View file

@ -190,7 +190,7 @@ func testLoadBlob(t *testing.T, version uint) {
defer cleanup()
length := 1000000
buf := restic.NewBlobBuffer(length)
buf := crypto.NewBlobBuffer(length)
_, err := io.ReadFull(rnd, buf)
rtest.OK(t, err)
@ -201,7 +201,7 @@ func testLoadBlob(t *testing.T, version uint) {
rtest.OK(t, err)
rtest.OK(t, repo.Flush(context.Background()))
base := restic.CiphertextLength(length)
base := crypto.CiphertextLength(length)
for _, testlength := range []int{0, base - 20, base - 1, base, base + 7, base + 15, base + 1000} {
buf = make([]byte, 0, testlength)
buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
@ -226,7 +226,7 @@ func benchmarkLoadBlob(b *testing.B, version uint) {
defer cleanup()
length := 1000000
buf := restic.NewBlobBuffer(length)
buf := crypto.NewBlobBuffer(length)
_, err := io.ReadFull(rnd, buf)
rtest.OK(b, err)
@ -269,7 +269,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) {
defer cleanup()
length := 1000000
buf := restic.NewBlobBuffer(length)
buf := crypto.NewBlobBuffer(length)
_, err := io.ReadFull(rnd, buf)
rtest.OK(b, err)

3
internal/restic/blob.go
View file

@ -3,6 +3,7 @@ package restic
import (
"fmt"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/errors"
)
@ -23,7 +24,7 @@ func (b Blob) DataLength() uint {
if b.UncompressedLength != 0 {
return b.UncompressedLength
}
return uint(PlaintextLength(int(b.Length)))
return uint(crypto.PlaintextLength(int(b.Length)))
}
func (b Blob) IsCompressed() bool {

2
internal/restorer/filerestorer_test.go
View file

@ -61,7 +61,7 @@ func newTestRepo(content []TestFile) *TestRepo {
key := crypto.NewRandomKey()
seal := func(data []byte) []byte {
ciphertext := restic.NewBlobBuffer(len(data))
ciphertext := crypto.NewBlobBuffer(len(data))
ciphertext = ciphertext[:0] // truncate the slice
nonce := crypto.NewRandomNonce()
ciphertext = append(ciphertext, nonce...)

2
internal/restic/hardlinks_index.go → internal/restorer/hardlinks_index.go
View file

@ -1,4 +1,4 @@
package restic
package restorer
import (
"sync"

6
internal/restic/hardlinks_index_test.go → internal/restorer/hardlinks_index_test.go
View file

@ -1,16 +1,16 @@
package restic_test
package restorer_test
import (
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/restorer"
rtest "github.com/restic/restic/internal/test"
)
// TestHardLinks contains various tests for HardlinkIndex.
func TestHardLinks(t *testing.T) {
idx := restic.NewHardlinkIndex()
idx := restorer.NewHardlinkIndex()
idx.Add(1, 2, "inode1-file1-on-device2")
idx.Add(2, 3, "inode2-file2-on-device3")

2
internal/restorer/restorer.go
View file

@ -218,7 +218,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
}
}
idx := restic.NewHardlinkIndex()
idx := NewHardlinkIndex()
filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, res.repo.Connections())
filerestorer.Error = res.Error