alexvanin/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

backend: split layout code into own subpackage

Browse source
This commit is contained in:
Michael Eischer 2022-10-15 16:23:39 +02:00
parent b361284f28
commit 4ccd5e806b
14 changed files with 43 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
internal/backend/azure/azure.go
View file

@ -13,6 +13,7 @@ import (
"strings" "strings"
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
@ -30,7 +31,7 @@ type Backend struct {
sem sema.Semaphore sem sema.Semaphore
prefix string prefix string
listMaxItems int listMaxItems int
backend.Layout layout.Layout
} }
const defaultListMaxItems = 5000 const defaultListMaxItems = 5000
@ -85,7 +86,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
connections: cfg.Connections, connections: cfg.Connections,
sem: sem, sem: sem,
prefix: cfg.Prefix, prefix: cfg.Prefix,
Layout: &backend.DefaultLayout{ Layout: &layout.DefaultLayout{
Path: cfg.Prefix, Path: cfg.Prefix,
Join: path.Join, Join: path.Join,
}, },

7
internal/backend/b2/b2.go
View file

@ -10,6 +10,7 @@ import (
"time" "time"
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
@ -25,7 +26,7 @@ type b2Backend struct {
bucket *b2.Bucket bucket *b2.Bucket
cfg Config cfg Config
listMaxItems int listMaxItems int
backend.Layout layout.Layout
sem sema.Semaphore sem sema.Semaphore
} }
@ -97,7 +98,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend
client: client, client: client,
bucket: bucket, bucket: bucket,
cfg: cfg, cfg: cfg,
Layout: &backend.DefaultLayout{ Layout: &layout.DefaultLayout{
Join: path.Join, Join: path.Join,
Path: cfg.Prefix, Path: cfg.Prefix,
}, },
@ -138,7 +139,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe
client: client, client: client,
bucket: bucket, bucket: bucket,
cfg: cfg, cfg: cfg,
Layout: &backend.DefaultLayout{ Layout: &layout.DefaultLayout{
Join: path.Join, Join: path.Join,
Path: cfg.Prefix, Path: cfg.Prefix,
}, },

5
internal/backend/gs/gs.go
View file

@ -14,6 +14,7 @@ import (
"cloud.google.com/go/storage" "cloud.google.com/go/storage"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -41,7 +42,7 @@ type Backend struct {
bucket *storage.BucketHandle bucket *storage.BucketHandle
prefix string prefix string
listMaxItems int listMaxItems int
backend.Layout layout.Layout
} }
// Ensure that *Backend implements restic.Backend. // Ensure that *Backend implements restic.Backend.
@ -111,7 +112,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
bucketName: cfg.Bucket, bucketName: cfg.Bucket,
bucket: gcsClient.Bucket(cfg.Bucket), bucket: gcsClient.Bucket(cfg.Bucket),
prefix: cfg.Prefix, prefix: cfg.Prefix,
Layout: &backend.DefaultLayout{ Layout: &layout.DefaultLayout{
Path: cfg.Prefix, Path: cfg.Prefix,
Join: path.Join, Join: path.Join,
}, },

2
internal/backend/layout.go → internal/backend/layout/layout.go
View file

@ -1,4 +1,4 @@
package backend package layout
import ( import (
"context" "context"

2
internal/backend/layout_default.go → internal/backend/layout/layout_default.go
View file

@ -1,4 +1,4 @@
package backend package layout
import ( import (
"encoding/hex" "encoding/hex"

2
internal/backend/layout_rest.go → internal/backend/layout/layout_rest.go
View file

@ -1,4 +1,4 @@
package backend package layout
import "github.com/restic/restic/internal/restic" import "github.com/restic/restic/internal/restic"

2
internal/backend/layout_s3legacy.go → internal/backend/layout/layout_s3legacy.go
View file

@ -1,4 +1,4 @@
package backend package layout
import "github.com/restic/restic/internal/restic" import "github.com/restic/restic/internal/restic"

16
internal/backend/layout_test.go → internal/backend/layout/layout_test.go
View file

@ -1,4 +1,4 @@
package backend package layout
import ( import (
"context" "context"
@ -362,15 +362,15 @@ func TestDetectLayout(t *testing.T) {
filename string filename string
want string want string
}{ }{
{"repo-layout-default.tar.gz", "*backend.DefaultLayout"}, {"repo-layout-default.tar.gz", "*layout.DefaultLayout"},
{"repo-layout-s3legacy.tar.gz", "*backend.S3LegacyLayout"}, {"repo-layout-s3legacy.tar.gz", "*layout.S3LegacyLayout"},
} }
var fs = &LocalFilesystem{} var fs = &LocalFilesystem{}
for _, test := range tests { for _, test := range tests {
for _, fs := range []Filesystem{fs, nil} { for _, fs := range []Filesystem{fs, nil} {
t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) { t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) {
rtest.SetupTarTestFixture(t, path, filepath.Join("testdata", test.filename)) rtest.SetupTarTestFixture(t, path, filepath.Join("../testdata", test.filename))
layout, err := DetectLayout(context.TODO(), fs, filepath.Join(path, "repo")) layout, err := DetectLayout(context.TODO(), fs, filepath.Join(path, "repo"))
if err != nil { if err != nil {
@ -401,12 +401,12 @@ func TestParseLayout(t *testing.T) {
defaultLayoutName string defaultLayoutName string
want string want string
}{ }{
{"default", "", "*backend.DefaultLayout"}, {"default", "", "*layout.DefaultLayout"},
{"s3legacy", "", "*backend.S3LegacyLayout"}, {"s3legacy", "", "*layout.S3LegacyLayout"},
{"", "", "*backend.DefaultLayout"}, {"", "", "*layout.DefaultLayout"},
} }
rtest.SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-default.tar.gz")) rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", "repo-layout-default.tar.gz"))
for _, test := range tests { for _, test := range tests {
t.Run(test.layoutName, func(t *testing.T) { t.Run(test.layoutName, func(t *testing.T) {

5
internal/backend/local/local.go
View file

@ -10,6 +10,7 @@ import (
"syscall" "syscall"
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
@ -23,7 +24,7 @@ import (
type Local struct { type Local struct {
Config Config
sem sema.Semaphore sem sema.Semaphore
backend.Layout layout.Layout
backend.Modes backend.Modes
} }
@ -33,7 +34,7 @@ var _ restic.Backend = &Local{}
const defaultLayout = "default" const defaultLayout = "default"
func open(ctx context.Context, cfg Config) (*Local, error) { func open(ctx context.Context, cfg Config) (*Local, error) {
l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) l, err := layout.ParseLayout(ctx, &layout.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
if err != nil { if err != nil {
return nil, err return nil, err
} }

6
internal/backend/rest/rest.go
View file

@ -14,7 +14,7 @@ import (
"strconv" "strconv"
"strings" "strings"
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
@ -32,7 +32,7 @@ type Backend struct {
connections uint connections uint
sem sema.Semaphore sem sema.Semaphore
client http.Client client http.Client
backend.Layout layout.Layout
} }
// the REST API protocol version is decided by HTTP request headers, these are the constants. // the REST API protocol version is decided by HTTP request headers, these are the constants.
@ -57,7 +57,7 @@ func Open(cfg Config, rt http.RoundTripper) (*Backend, error) {
be := &Backend{ be := &Backend{
url: cfg.URL, url: cfg.URL,
client: http.Client{Transport: rt}, client: http.Client{Transport: rt},
Layout: &backend.RESTLayout{URL: url, Join: path.Join}, Layout: &layout.RESTLayout{URL: url, Join: path.Join},
connections: cfg.Connections, connections: cfg.Connections,
sem: sem, sem: sem,
} }

7
internal/backend/s3/s3.go
View file

@ -13,6 +13,7 @@ import (
"time" "time"
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
@ -28,7 +29,7 @@ type Backend struct {
client *minio.Client client *minio.Client
sem sema.Semaphore sem sema.Semaphore
cfg Config cfg Config
backend.Layout layout.Layout
} }
// make sure that *Backend implements backend.Backend // make sure that *Backend implements backend.Backend
@ -113,7 +114,7 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
cfg: cfg, cfg: cfg,
} }
l, err := backend.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix) l, err := layout.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -514,7 +515,7 @@ func (be *Backend) Delete(ctx context.Context) error {
func (be *Backend) Close() error { return nil } func (be *Backend) Close() error { return nil }
// Rename moves a file based on the new layout l. // Rename moves a file based on the new layout l.
func (be *Backend) Rename(ctx context.Context, h restic.Handle, l backend.Layout) error { func (be *Backend) Rename(ctx context.Context, h restic.Handle, l layout.Layout) error {
debug.Log("Rename %v to %v", h, l) debug.Log("Rename %v to %v", h, l)
oldname := be.Filename(h) oldname := be.Filename(h)
newname := l.Filename(h) newname := l.Filename(h)

7
internal/backend/sftp/sftp.go
View file

@ -14,6 +14,7 @@ import (
"time" "time"
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
@ -35,7 +36,7 @@ type SFTP struct {
posixRename bool posixRename bool
sem sema.Semaphore sem sema.Semaphore
backend.Layout layout.Layout
Config Config
backend.Modes backend.Modes
} }
@ -144,7 +145,7 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) {
return nil, err return nil, err
} }
sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -243,7 +244,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
return nil, err return nil, err
} }
sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
if err != nil { if err != nil {
return nil, err return nil, err
} }

5
internal/backend/swift/swift.go
View file

@ -14,6 +14,7 @@ import (
"time" "time"
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
@ -30,7 +31,7 @@ type beSwift struct {
sem sema.Semaphore sem sema.Semaphore
container string // Container name container string // Container name
prefix string // Prefix of object names in the container prefix string // Prefix of object names in the container
backend.Layout layout.Layout
} }
// ensure statically that *beSwift implements restic.Backend. // ensure statically that *beSwift implements restic.Backend.
@ -74,7 +75,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend
sem: sem, sem: sem,
container: cfg.Container, container: cfg.Container,
prefix: cfg.Prefix, prefix: cfg.Prefix,
Layout: &backend.DefaultLayout{ Layout: &layout.DefaultLayout{
Path: cfg.Prefix, Path: cfg.Prefix,
Join: path.Join, Join: path.Join,
}, },

8
internal/migrations/s3_layout.go
View file

@ -6,7 +6,7 @@ import (
"os" "os"
"path" "path"
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/cache"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
@ -74,7 +74,7 @@ func retry(max int, fail func(err error), f func() error) error {
// maxErrors for retrying renames on s3. // maxErrors for retrying renames on s3.
const maxErrors = 20 const maxErrors = 20
func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layout, t restic.FileType) error { func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layout, t restic.FileType) error {
printErr := func(err error) { printErr := func(err error) {
fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err) fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err)
} }
@ -97,12 +97,12 @@ func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error {
return errors.New("backend is not s3") return errors.New("backend is not s3")
} }
oldLayout := &backend.S3LegacyLayout{ oldLayout := &layout.S3LegacyLayout{
Path: be.Path(), Path: be.Path(),
Join: path.Join, Join: path.Join,
} }
newLayout := &backend.DefaultLayout{ newLayout := &layout.DefaultLayout{
Path: be.Path(), Path: be.Path(),
Join: path.Join, Join: path.Join,
} }