Merge pull request #957 from middelink/fix-953

Change backup policy to be inclusive
This commit is contained in:
Alexander Neumann 2017-05-15 17:19:04 +02:00
commit 13393c76dc
15 changed files with 122 additions and 315 deletions

View file

@ -788,6 +788,9 @@ All the ``--keep-*`` options above only count
hours/days/weeks/months/years which have a snapshot, so those without a
snapshot are ignored.
All snapshots are evaluated counted against all matching keep-* counts. A
single snapshot on 2017-09-30 (Sun) will count as a daily, weekly and monthly.
Let's explain this with an example: Suppose you have only made a backup
on each Sunday for 12 weeks. Then ``forget --keep-daily 4`` will keep
the last four snapshots for the last four Sundays, but remove the rest.
@ -796,6 +799,14 @@ is a safety feature: it prevents restic from removing many snapshots
when no new ones are created. If it was implemented otherwise, running
``forget --keep-daily 4`` on a Friday would remove all snapshots!
Another example: Suppose you make daily backups for 100 years. Then
``forget --keep-daily 7 --keep-weekly 5 --keep-monthly 12 --keep-yearly 75``
will keep the most recent 7 daily snapshots, then 4 (remember, 7 dailies
already include a week!) last-day-of-the-weeks and 11 or 12
last-day-of-the-months (11 or 12 depends if the 5 weeklies cross a month).
And finally 75 last-day-of-the-year snapshots. All other snapshots are
removed.
Autocompletion
--------------

View file

@ -1,7 +1,6 @@
package restic
import (
"fmt"
"reflect"
"sort"
"time"
@ -25,40 +24,6 @@ func (sn Snapshots) Swap(i, j int) {
sn[i], sn[j] = sn[j], sn[i]
}
// SnapshotFilter configures criteria for filtering snapshots before an
// ExpirePolicy can be applied.
type SnapshotFilter struct {
Hostname string
Username string
Paths []string
Tags []string
}
// FilterSnapshots returns the snapshots from s which match the filter f.
func FilterSnapshots(s Snapshots, f SnapshotFilter) (result Snapshots) {
for _, snap := range s {
if f.Hostname != "" && f.Hostname != snap.Hostname {
continue
}
if f.Username != "" && f.Username != snap.Username {
continue
}
if f.Paths != nil && !reflect.DeepEqual(f.Paths, snap.Paths) {
continue
}
if !snap.HasTags(f.Tags) {
continue
}
result = append(result, snap)
}
return result
}
// ExpirePolicy configures which snapshots should be automatically removed.
type ExpirePolicy struct {
Last int // keep the last n snapshots
@ -86,18 +51,6 @@ func (e ExpirePolicy) Empty() bool {
return reflect.DeepEqual(e, empty)
}
// filter is used to split a list of snapshots into those to keep and those to
// remove according to a policy.
type filter struct {
Unprocessed Snapshots
Remove Snapshots
Keep Snapshots
}
func (f filter) String() string {
return fmt.Sprintf("<filter %d todo, %d keep, %d remove>", len(f.Unprocessed), len(f.Keep), len(f.Remove))
}
// ymdh returns an integer in the form YYYYMMDDHH.
func ymdh(d time.Time) int {
return d.Year()*1000000 + int(d.Month())*10000 + d.Day()*100 + d.Hour()
@ -124,84 +77,13 @@ func y(d time.Time) int {
return d.Year()
}
// apply moves snapshots from Unprocess to either Keep or Remove. It sorts the
// snapshots into buckets according to the return value of fn, and then moves
// the newest snapshot in each bucket to Keep and all others to Remove. When
// max snapshots were found, processing stops.
func (f *filter) apply(fn func(time.Time) int, max int) {
if max == 0 || len(f.Unprocessed) == 0 {
return
}
sameBucket := Snapshots{}
lastBucket := fn(f.Unprocessed[0].Time)
for len(f.Unprocessed) > 0 {
cur := f.Unprocessed[0]
bucket := fn(cur.Time)
// if the snapshots are from a new bucket, forget all but the first
// (=last in time) snapshot from the previous bucket.
if bucket != lastBucket {
f.Keep = append(f.Keep, sameBucket[0])
f.Remove = append(f.Remove, sameBucket[1:]...)
sameBucket = Snapshots{}
lastBucket = bucket
max--
if max == 0 {
return
}
}
// collect all snapshots for the current bucket
sameBucket = append(sameBucket, cur)
f.Unprocessed = f.Unprocessed[1:]
}
// if we have leftovers, process them too.
if len(sameBucket) > 0 {
f.Keep = append(f.Keep, sameBucket[0])
f.Remove = append(f.Remove, sameBucket[1:]...)
}
// always returns a unique number for d.
func always(d time.Time) int {
return int(d.UnixNano())
}
// keepTags marks the snapshots which have all tags as to be kept.
func (f *filter) keepTags(tags []string) {
if len(tags) == 0 {
return
}
unprocessed := f.Unprocessed[:0]
for _, sn := range f.Unprocessed {
if sn.HasTags(tags) {
f.Keep = append(f.Keep, sn)
continue
}
unprocessed = append(unprocessed, sn)
}
f.Unprocessed = unprocessed
}
// keepLast marks the last n snapshots as to be kept.
func (f *filter) keepLast(n int) {
if n > len(f.Unprocessed) {
n = len(f.Unprocessed)
}
f.Keep = append(f.Keep, f.Unprocessed[:n]...)
f.Unprocessed = f.Unprocessed[n:]
}
// finish moves all remaining snapshots to remove.
func (f *filter) finish() {
f.Remove = append(f.Remove, f.Unprocessed...)
}
// ApplyPolicy runs returns the snapshots from s that are to be deleted according
// to the policy p. s is sorted in the process.
// ApplyPolicy returns the snapshots from list that are to be kept and removed
// according to the policy p. list is sorted in the process.
func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) {
sort.Sort(list)
@ -213,20 +95,46 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) {
return list, remove
}
f := filter{
Unprocessed: list,
Remove: Snapshots{},
Keep: Snapshots{},
var buckets = [6]struct {
Count int
bucker func(d time.Time) int
Last int
}{
{p.Last, always, -1},
{p.Hourly, ymdh, -1},
{p.Daily, ymd, -1},
{p.Weekly, yw, -1},
{p.Monthly, ym, -1},
{p.Yearly, y, -1},
}
f.keepTags(p.Tags)
f.keepLast(p.Last)
f.apply(ymdh, p.Hourly)
f.apply(ymd, p.Daily)
f.apply(yw, p.Weekly)
f.apply(ym, p.Monthly)
f.apply(y, p.Yearly)
f.finish()
for _, cur := range list {
var keepSnap bool
return f.Keep, f.Remove
// Tags are handled specially as they are not counted.
if len(p.Tags) > 0 {
if cur.HasTags(p.Tags) {
keepSnap = true
}
}
// Now update the other buckets and see if they have some counts left.
for i, b := range buckets {
if b.Count > 0 {
val := b.bucker(cur.Time)
if val != b.Last {
keepSnap = true
buckets[i].Last = val
buckets[i].Count--
}
}
}
if keepSnap {
keep = append(keep, cur)
} else {
remove = append(remove, cur)
}
}
return keep, remove
}

View file

@ -7,7 +7,6 @@ import (
"path/filepath"
"reflect"
"restic"
"sort"
"testing"
"time"
)
@ -21,76 +20,6 @@ func parseTimeUTC(s string) time.Time {
return t.UTC()
}
var testFilterSnapshots = restic.Snapshots{
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-01 01:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}},
{Hostname: "bar", Username: "testuser", Time: parseTimeUTC("2016-01-01 01:03:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-03 07:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}},
{Hostname: "bar", Username: "testuser", Time: parseTimeUTC("2016-01-01 07:08:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 10:23:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 11:23:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 12:23:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 12:24:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"test"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 12:28:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"test"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 12:30:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"test", "foo", "bar"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 16:23:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"test", "test2"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-05 09:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-06 08:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"fox"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-07 10:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"fox"}},
{Hostname: "foo", Username: "root", Time: parseTimeUTC("2016-01-08 20:02:03"), Paths: []string{"/usr", "/sbin"}, Tags: []string{"foo"}},
{Hostname: "foo", Username: "root", Time: parseTimeUTC("2016-01-09 21:02:03"), Paths: []string{"/usr", "/sbin"}, Tags: []string{"fox"}},
{Hostname: "bar", Username: "root", Time: parseTimeUTC("2016-01-12 21:02:03"), Paths: []string{"/usr", "/sbin"}, Tags: []string{"foo"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-12 21:08:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"bar"}},
{Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-18 12:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"bar"}},
}
var filterTests = []restic.SnapshotFilter{
{Hostname: "foo"},
{Username: "root"},
{Hostname: "foo", Username: "root"},
{Paths: []string{"/usr", "/bin"}},
{Hostname: "bar", Paths: []string{"/usr", "/bin"}},
{Hostname: "foo", Username: "root", Paths: []string{"/usr", "/sbin"}},
{Tags: []string{"foo"}},
{Tags: []string{"fox"}, Username: "root"},
{Tags: []string{"foo", "test"}},
{Tags: []string{"foo", "test2"}},
}
func TestFilterSnapshots(t *testing.T) {
sort.Sort(testFilterSnapshots)
for i, f := range filterTests {
res := restic.FilterSnapshots(testFilterSnapshots, f)
goldenFilename := filepath.Join("testdata", fmt.Sprintf("filter_snapshots_%d", i))
if *updateGoldenFiles {
buf, err := json.MarshalIndent(res, "", " ")
if err != nil {
t.Fatalf("error marshaling result: %v", err)
}
if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil {
t.Fatalf("unable to update golden file: %v", err)
}
}
buf, err := ioutil.ReadFile(goldenFilename)
if err != nil {
t.Errorf("error loading golden file %v: %v", goldenFilename, err)
continue
}
var want restic.Snapshots
err = json.Unmarshal(buf, &want)
if !reflect.DeepEqual(res, want) {
t.Errorf("test %v: wrong result, want:\n %#v\ngot:\n %#v", i, want, res)
continue
}
}
}
func TestExpireSnapshotOps(t *testing.T) {
data := []struct {
expectEmpty bool
@ -128,7 +57,7 @@ var testExpireSnapshots = restic.Snapshots{
{Time: parseTimeUTC("2014-08-10 10:20:30")},
{Time: parseTimeUTC("2014-08-12 10:20:30")},
{Time: parseTimeUTC("2014-08-13 10:20:30")},
{Time: parseTimeUTC("2014-08-13 10:20:30")},
{Time: parseTimeUTC("2014-08-13 10:20:30.1")},
{Time: parseTimeUTC("2014-08-15 10:20:30")},
{Time: parseTimeUTC("2014-08-18 10:20:30")},
{Time: parseTimeUTC("2014-08-20 10:20:30")},
@ -148,8 +77,8 @@ var testExpireSnapshots = restic.Snapshots{
{Time: parseTimeUTC("2014-11-10 10:20:30"), Tags: []string{"foo"}},
{Time: parseTimeUTC("2014-11-12 10:20:30"), Tags: []string{"foo"}},
{Time: parseTimeUTC("2014-11-13 10:20:30"), Tags: []string{"foo"}},
{Time: parseTimeUTC("2014-11-13 10:20:30")},
{Time: parseTimeUTC("2014-11-15 10:20:30")},
{Time: parseTimeUTC("2014-11-13 10:20:30.1"), Tags: []string{"bar"}},
{Time: parseTimeUTC("2014-11-15 10:20:30"), Tags: []string{"foo", "bar"}},
{Time: parseTimeUTC("2014-11-18 10:20:30")},
{Time: parseTimeUTC("2014-11-20 10:20:30")},
{Time: parseTimeUTC("2014-11-21 10:20:30")},
@ -168,7 +97,7 @@ var testExpireSnapshots = restic.Snapshots{
{Time: parseTimeUTC("2015-08-10 10:20:30")},
{Time: parseTimeUTC("2015-08-12 10:20:30")},
{Time: parseTimeUTC("2015-08-13 10:20:30")},
{Time: parseTimeUTC("2015-08-13 10:20:30")},
{Time: parseTimeUTC("2015-08-13 10:20:30.1")},
{Time: parseTimeUTC("2015-08-15 10:20:30")},
{Time: parseTimeUTC("2015-08-18 10:20:30")},
{Time: parseTimeUTC("2015-08-20 10:20:30")},
@ -188,7 +117,7 @@ var testExpireSnapshots = restic.Snapshots{
{Time: parseTimeUTC("2015-11-10 10:20:30")},
{Time: parseTimeUTC("2015-11-12 10:20:30")},
{Time: parseTimeUTC("2015-11-13 10:20:30")},
{Time: parseTimeUTC("2015-11-13 10:20:30")},
{Time: parseTimeUTC("2015-11-13 10:20:30.1")},
{Time: parseTimeUTC("2015-11-15 10:20:30")},
{Time: parseTimeUTC("2015-11-18 10:20:30")},
{Time: parseTimeUTC("2015-11-20 10:20:30")},
@ -235,6 +164,7 @@ var expireTests = []restic.ExpirePolicy{
{Yearly: 10},
{Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10},
{Tags: []string{"foo"}},
{Tags: []string{"foo", "bar"}},
}
func TestApplyPolicy(t *testing.T) {

View file

@ -120,7 +120,7 @@
"paths": null
},
{
"time": "2015-11-13T10:20:30Z",
"time": "2015-11-13T10:20:30.1Z",
"tree": null,
"paths": null
},
@ -270,7 +270,7 @@
"paths": null
},
{
"time": "2015-08-13T10:20:30Z",
"time": "2015-08-13T10:20:30.1Z",
"tree": null,
"paths": null
},
@ -317,12 +317,19 @@
{
"time": "2014-11-15T10:20:30Z",
"tree": null,
"paths": null
"paths": null,
"tags": [
"foo",
"bar"
]
},
{
"time": "2014-11-13T10:20:30Z",
"time": "2014-11-13T10:20:30.1Z",
"tree": null,
"paths": null
"paths": null,
"tags": [
"bar"
]
},
{
"time": "2014-11-13T10:20:30Z",
@ -512,7 +519,7 @@
"paths": null
},
{
"time": "2014-08-13T10:20:30Z",
"time": "2014-08-13T10:20:30.1Z",
"tree": null,
"paths": null
},

View file

@ -9,11 +9,6 @@
"tree": null,
"paths": null
},
{
"time": "2016-01-12T21:02:03Z",
"tree": null,
"paths": null
},
{
"time": "2016-01-09T21:02:03Z",
"tree": null,
@ -53,10 +48,5 @@
"time": "2016-01-01T07:08:03Z",
"tree": null,
"paths": null
},
{
"time": "2015-11-22T10:20:30Z",
"tree": null,
"paths": null
}
]

View file

@ -14,24 +14,9 @@
"tree": null,
"paths": null
},
{
"time": "2016-01-08T20:02:03Z",
"tree": null,
"paths": null
},
{
"time": "2016-01-03T07:02:03Z",
"tree": null,
"paths": null
},
{
"time": "2015-11-22T10:20:30Z",
"tree": null,
"paths": null
},
{
"time": "2015-11-15T10:20:30Z",
"tree": null,
"paths": null
}
]

View file

@ -9,16 +9,6 @@
"tree": null,
"paths": null
},
{
"time": "2016-01-09T21:02:03Z",
"tree": null,
"paths": null
},
{
"time": "2016-01-03T07:02:03Z",
"tree": null,
"paths": null
},
{
"time": "2015-11-22T10:20:30Z",
"tree": null,
@ -43,13 +33,5 @@
"time": "2014-11-22T10:20:30Z",
"tree": null,
"paths": null
},
{
"time": "2014-10-22T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo"
]
}
]

View file

@ -34,16 +34,6 @@
"tree": null,
"paths": null
},
{
"time": "2016-01-04T16:23:03Z",
"tree": null,
"paths": null
},
{
"time": "2016-01-03T07:02:03Z",
"tree": null,
"paths": null
},
{
"time": "2015-11-22T10:20:30Z",
"tree": null,
@ -54,16 +44,6 @@
"tree": null,
"paths": null
},
{
"time": "2015-09-22T10:20:30Z",
"tree": null,
"paths": null
},
{
"time": "2015-08-22T10:20:30Z",
"tree": null,
"paths": null
},
{
"time": "2014-11-22T10:20:30Z",
"tree": null,

View file

@ -1,4 +1,13 @@
[
{
"time": "2014-11-15T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo",
"bar"
]
},
{
"time": "2014-11-13T10:20:30Z",
"tree": null,

View file

@ -0,0 +1,11 @@
[
{
"time": "2014-11-15T10:20:30Z",
"tree": null,
"paths": null,
"tags": [
"foo",
"bar"
]
}
]

View file

@ -120,7 +120,7 @@
"paths": null
},
{
"time": "2015-11-13T10:20:30Z",
"time": "2015-11-13T10:20:30.1Z",
"tree": null,
"paths": null
},
@ -270,7 +270,7 @@
"paths": null
},
{
"time": "2015-08-13T10:20:30Z",
"time": "2015-08-13T10:20:30.1Z",
"tree": null,
"paths": null
},
@ -317,12 +317,19 @@
{
"time": "2014-11-15T10:20:30Z",
"tree": null,
"paths": null
"paths": null,
"tags": [
"foo",
"bar"
]
},
{
"time": "2014-11-13T10:20:30Z",
"time": "2014-11-13T10:20:30.1Z",
"tree": null,
"paths": null
"paths": null,
"tags": [
"bar"
]
},
{
"time": "2014-11-13T10:20:30Z",
@ -512,7 +519,7 @@
"paths": null
},
{
"time": "2014-08-13T10:20:30Z",
"time": "2014-08-13T10:20:30.1Z",
"tree": null,
"paths": null
},

View file

@ -120,7 +120,7 @@
"paths": null
},
{
"time": "2015-11-13T10:20:30Z",
"time": "2015-11-13T10:20:30.1Z",
"tree": null,
"paths": null
},
@ -270,7 +270,7 @@
"paths": null
},
{
"time": "2015-08-13T10:20:30Z",
"time": "2015-08-13T10:20:30.1Z",
"tree": null,
"paths": null
},
@ -317,12 +317,19 @@
{
"time": "2014-11-15T10:20:30Z",
"tree": null,
"paths": null
"paths": null,
"tags": [
"foo",
"bar"
]
},
{
"time": "2014-11-13T10:20:30Z",
"time": "2014-11-13T10:20:30.1Z",
"tree": null,
"paths": null
"paths": null,
"tags": [
"bar"
]
},
{
"time": "2014-11-13T10:20:30Z",
@ -512,7 +519,7 @@
"paths": null
},
{
"time": "2014-08-13T10:20:30Z",
"time": "2014-08-13T10:20:30.1Z",
"tree": null,
"paths": null
},

View file

@ -95,7 +95,7 @@
"paths": null
},
{
"time": "2015-11-13T10:20:30Z",
"time": "2015-11-13T10:20:30.1Z",
"tree": null,
"paths": null
}

View file

@ -75,7 +75,7 @@
"paths": null
},
{
"time": "2015-11-13T10:20:30Z",
"time": "2015-11-13T10:20:30.1Z",
"tree": null,
"paths": null
},

View file

@ -28,25 +28,5 @@
"time": "2016-01-07T10:02:03Z",
"tree": null,
"paths": null
},
{
"time": "2016-01-06T08:02:03Z",
"tree": null,
"paths": null
},
{
"time": "2016-01-05T09:02:03Z",
"tree": null,
"paths": null
},
{
"time": "2016-01-04T16:23:03Z",
"tree": null,
"paths": null
},
{
"time": "2016-01-03T07:02:03Z",
"tree": null,
"paths": null
}
]