Merge pull request #230 from restic/refresh-locks

Refresh locks

cmd/restic/lock.go

@@ -4,14 +4,21 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
+	"time"
 
 	"github.com/restic/restic"
 	"github.com/restic/restic/debug"
 	"github.com/restic/restic/repository"
 )
 
-var globalLocks []*restic.Lock
+var globalLocks struct {
+	locks         []*restic.Lock
+	cancelRefresh chan struct{}
+	refreshWG     sync.WaitGroup
+	sync.Mutex
+}
 
 func lockRepo(repo *repository.Repository) (*restic.Lock, error) {
 	return lockRepository(repo, false)
@@ -29,35 +36,69 @@ func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock,
 
 	lock, err := lockFn(repo)
 	if err != nil {
-		if restic.IsAlreadyLocked(err) {
-			tpe := ""
-			if exclusive {
-				tpe = " exclusive"
-			}
-			fmt.Fprintf(os.Stderr, "unable to acquire%s lock for operation:\n", tpe)
-			fmt.Fprintln(os.Stderr, err)
-			fmt.Fprintf(os.Stderr, "\nthe `unlock` command can be used to remove stale locks\n")
-			os.Exit(1)
-		}
-
 		return nil, err
 	}
 
-	globalLocks = append(globalLocks, lock)
+	globalLocks.Lock()
+	if globalLocks.cancelRefresh == nil {
+		debug.Log("main.lockRepository", "start goroutine for lock refresh")
+		globalLocks.cancelRefresh = make(chan struct{})
+		globalLocks.refreshWG = sync.WaitGroup{}
+		globalLocks.refreshWG.Add(1)
+		go refreshLocks(&globalLocks.refreshWG, globalLocks.cancelRefresh)
+	}
+
+	globalLocks.locks = append(globalLocks.locks, lock)
+	globalLocks.Unlock()
 
 	return lock, err
 }
 
+var refreshInterval = 5 * time.Minute
+
+func refreshLocks(wg *sync.WaitGroup, done <-chan struct{}) {
+	debug.Log("main.refreshLocks", "start")
+	defer func() {
+		wg.Done()
+		globalLocks.Lock()
+		globalLocks.cancelRefresh = nil
+		globalLocks.Unlock()
+	}()
+
+	ticker := time.NewTicker(refreshInterval)
+
+	for {
+		select {
+		case <-done:
+			debug.Log("main.refreshLocks", "terminate")
+			return
+		case <-ticker.C:
+			debug.Log("main.refreshLocks", "refreshing locks")
+			globalLocks.Lock()
+			for _, lock := range globalLocks.locks {
+				err := lock.Refresh()
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "unable to refresh lock: %v\n", err)
+				}
+			}
+			globalLocks.Unlock()
+		}
+	}
+}
+
 func unlockRepo(lock *restic.Lock) error {
+	globalLocks.Lock()
+	defer globalLocks.Unlock()
+
 	debug.Log("unlockRepo", "unlocking repository")
 	if err := lock.Unlock(); err != nil {
 		debug.Log("unlockRepo", "error while unlocking: %v", err)
 		return err
 	}
 
-	for i := 0; i < len(globalLocks); i++ {
-		if lock == globalLocks[i] {
-			globalLocks = append(globalLocks[:i], globalLocks[i+1:]...)
+	for i := 0; i < len(globalLocks.locks); i++ {
+		if lock == globalLocks.locks[i] {
+			globalLocks.locks = append(globalLocks.locks[:i], globalLocks.locks[i+1:]...)
 			return nil
 		}
 	}
@@ -66,8 +107,11 @@ func unlockRepo(lock *restic.Lock) error {
 }
 
 func unlockAll() error {
-	debug.Log("unlockAll", "unlocking %d locks", len(globalLocks))
-	for _, lock := range globalLocks {
+	globalLocks.Lock()
+	defer globalLocks.Unlock()
+
+	debug.Log("unlockAll", "unlocking %d locks", len(globalLocks.locks))
+	for _, lock := range globalLocks.locks {
 		if err := lock.Unlock(); err != nil {
 			debug.Log("unlockAll", "error while unlocking: %v", err)
 			return err

cmd/restic/main.go

@@ -1,10 +1,12 @@
 package main
 
 import (
+	"fmt"
 	"os"
 	"runtime"
 
 	"github.com/jessevdk/go-flags"
+	"github.com/restic/restic"
 	"github.com/restic/restic/debug"
 )
 
@@ -26,6 +28,10 @@ func main() {
 		os.Exit(0)
 	}
 
+	if restic.IsAlreadyLocked(err) {
+		fmt.Fprintf(os.Stderr, "\nthe `unlock` command can be used to remove stale locks\n")
+	}
+
 	if err != nil {
 		os.Exit(1)
 	}

lock.go

@@ -20,6 +20,9 @@ import (
 // There are two types of locks: exclusive and non-exclusive. There may be many
 // different non-exclusive locks, but at most one exclusive lock, which can
 // only be acquired while no non-exclusive lock is held.
+//
+// A lock must be refreshed regularly to not be considered stale, this must be
+// triggered by regularly calling Refresh.
 type Lock struct {
 	Time      time.Time `json:"time"`
 	Exclusive bool      `json:"exclusive"`
@@ -89,7 +92,7 @@ func newLock(repo *repository.Repository, excl bool) (*Lock, error) {
 		return nil, err
 	}
 
-	err = lock.createLock()
+	lock.lockID, err = lock.createLock()
 	if err != nil {
 		return nil, err
 	}
@@ -171,14 +174,13 @@ func eachLock(repo *repository.Repository, f func(backend.ID, *Lock, error) erro
 }
 
 // createLock acquires the lock by creating a file in the repository.
-func (l *Lock) createLock() error {
+func (l *Lock) createLock() (backend.ID, error) {
 	id, err := l.repo.SaveJSONUnpacked(backend.Lock, l)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	l.lockID = id
-	return nil
+	return id, nil
 }
 
 // Unlock removes the lock from the repository.
@@ -220,6 +222,26 @@ func (l *Lock) Stale() bool {
 	return false
 }
 
+// Refresh refreshes the lock by creating a new file in the backend with a new
+// timestamp. Afterwards the old lock is removed.
+func (l *Lock) Refresh() error {
+	debug.Log("Lock.Refresh", "refreshing lock %v", l.lockID.Str())
+	id, err := l.createLock()
+	if err != nil {
+		return err
+	}
+
+	err = l.repo.Backend().Remove(backend.Lock, l.lockID.String())
+	if err != nil {
+		return err
+	}
+
+	debug.Log("Lock.Refresh", "new lock ID %v", id.Str())
+	l.lockID = id
+
+	return nil
+}
+
 func (l Lock) String() string {
 	text := fmt.Sprintf("PID %d on %s by %s (UID %d, GID %d)\nlock was created at %s (%s ago)\nstorage ID %v",
 		l.PID, l.Hostname, l.Username, l.UID, l.GID,
@@ -275,6 +297,7 @@ func RemoveStaleLocks(repo *repository.Repository) error {
 	})
 }
 
+// RemoveAllLocks removes all locks forcefully.
 func RemoveAllLocks(repo *repository.Repository) error {
 	return eachLock(repo, func(id backend.ID, lock *Lock, err error) error {
 		return repo.Backend().Remove(backend.Lock, id.String())

lock_test.go

@@ -195,3 +195,33 @@ func TestRemoveAllLocks(t *testing.T) {
 	Assert(t, lockExists(repo, t, id3) == false,
 		"lock still exists after RemoveAllLocks was called")
 }
+
+func TestLockRefresh(t *testing.T) {
+	repo := SetupRepo()
+	defer TeardownRepo(repo)
+
+	lock, err := restic.NewLock(repo)
+	OK(t, err)
+
+	var lockID backend.ID
+	for id := range repo.List(backend.Lock, nil) {
+		if lockID != nil {
+			t.Error("more than one lock found")
+		}
+		lockID = id
+	}
+
+	OK(t, lock.Refresh())
+
+	var lockID2 backend.ID
+	for id := range repo.List(backend.Lock, nil) {
+		if lockID2 != nil {
+			t.Error("more than one lock found")
+		}
+		lockID2 = id
+	}
+
+	Assert(t, !lockID.Equal(lockID2),
+		"expected a new ID after lock refresh, got the same")
+	OK(t, lock.Unlock())
+}