restic/repository.go

package khepri

import (
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
)

const (
	dirMode        = 0700
	blobPath       = "blobs"
	refPath        = "refs"
	tempPath       = "tmp"
	configFileName = "config.json"
)

var (
	ErrIDDoesNotExist = errors.New("ID does not exist")
)

// Name stands for the alias given to an ID.
type Name string

func (n Name) Encode() string {
	return url.QueryEscape(string(n))
}

type HashFunc func() hash.Hash

type Repository struct {
	path   string
	hash   HashFunc
	config *Config
}

type Config struct {
	Salt string
	N    uint
	R    uint `json:"r"`
	P    uint `json:"p"`
}

// TODO: figure out scrypt values on the fly depending on the current
// hardware.
const (
	scrypt_N        = 65536
	scrypt_r        = 8
	scrypt_p        = 1
	scrypt_saltsize = 64
)

type Type int

const (
	TYPE_BLOB = iota
	TYPE_REF
)

func NewTypeFromString(s string) Type {
	switch s {
	case "blob":
		return TYPE_BLOB
	case "ref":
		return TYPE_REF
	}

	panic(fmt.Sprintf("unknown type %q", s))
}

func (t Type) String() string {
	switch t {
	case TYPE_BLOB:
		return "blob"
	case TYPE_REF:
		return "ref"
	}

	panic(fmt.Sprintf("unknown type %d", t))
}

// NewRepository opens a dir-baked repository at the given path.
func NewRepository(path string) (*Repository, error) {
	var err error

	d := &Repository{
		path: path,
		hash: sha256.New,
	}

	d.config, err = d.read_config()
	if err != nil {
		return nil, err
	}

	return d, nil
}

func (r *Repository) read_config() (*Config, error) {
	// try to open config file
	f, err := os.Open(path.Join(r.path, configFileName))
	if err != nil {
		return nil, err
	}

	cfg := new(Config)
	buf, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(buf, cfg)
	if err != nil {
		return nil, err
	}

	return cfg, nil
}

// CreateRepository creates all the necessary files and directories for the
// Repository.
func CreateRepository(p string) (*Repository, error) {
	dirs := []string{
		p,
		path.Join(p, blobPath),
		path.Join(p, refPath),
		path.Join(p, tempPath),
	}

	var configfile = path.Join(p, configFileName)

	// test if repository directories or config file already exist
	if _, err := os.Stat(configfile); err == nil {
		return nil, fmt.Errorf("config file %s already exists", configfile)
	}

	for _, d := range dirs[1:] {
		if _, err := os.Stat(d); err == nil {
			return nil, fmt.Errorf("dir %s already exists", d)
		}
	}

	// create initial json configuration
	cfg := &Config{
		N: scrypt_N,
		R: scrypt_r,
		P: scrypt_p,
	}

	// generate salt
	buf := make([]byte, scrypt_saltsize)
	n, err := rand.Read(buf)
	if n != scrypt_saltsize || err != nil {
		panic("unable to read enough random bytes for salt")
	}
	cfg.Salt = hex.EncodeToString(buf)

	// create ps for blobs, refs and temp
	for _, dir := range dirs {
		err := os.MkdirAll(dir, dirMode)
		if err != nil {
			return nil, err
		}
	}

	// write config file
	f, err := os.Create(configfile)
	defer f.Close()
	if err != nil {
		return nil, err
	}

	s, err := json.Marshal(cfg)
	if err != nil {
		return nil, err
	}

	_, err = f.Write(s)
	if err != nil {
		return nil, err
	}

	// open repository
	return NewRepository(p)
}

// SetHash changes the hash function used for deriving IDs. Default is SHA256.
func (r *Repository) SetHash(h HashFunc) {
	r.hash = h
}

// Path returns the directory used for this repository.
func (r *Repository) Path() string {
	return r.path
}

// Return temp directory in correct directory for this repository.
func (r *Repository) tempFile() (*os.File, error) {
	return ioutil.TempFile(path.Join(r.path, tempPath), "temp-")
}

// Rename temp file to final name according to type and ID.
func (r *Repository) renameFile(file *os.File, t Type, id ID) error {
	filename := path.Join(r.dir(t), id.String())
	return os.Rename(file.Name(), filename)
}

// Construct directory for given Type.
func (r *Repository) dir(t Type) string {
	switch t {
	case TYPE_BLOB:
		return path.Join(r.path, blobPath)
	case TYPE_REF:
		return path.Join(r.path, refPath)
	}

	panic(fmt.Sprintf("unknown type %d", t))
}

// Construct path for given Type and ID.
func (r *Repository) filename(t Type, id ID) string {
	return path.Join(r.dir(t), id.String())
}

// Test returns true if the given ID exists in the repository.
func (r *Repository) Test(t Type, id ID) (bool, error) {
	// try to open file
	file, err := os.Open(r.filename(t, id))
	defer func() {
		file.Close()
	}()

	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

// Get returns a reader for the content stored under the given ID.
func (r *Repository) Get(t Type, id ID) (io.ReadCloser, error) {
	// try to open file
	file, err := os.Open(r.filename(t, id))
	if err != nil {
		return nil, err
	}

	return file, nil
}

// Remove removes the content stored at ID.
func (r *Repository) Remove(t Type, id ID) error {
	return os.Remove(r.filename(t, id))
}

type IDs []ID

// Lists all objects of a given type.
func (r *Repository) List(t Type) (IDs, error) {
	// TODO: use os.Open() and d.Readdirnames() instead of Glob()
	pattern := path.Join(r.dir(t), "*")

	matches, err := filepath.Glob(pattern)
	if err != nil {
		return nil, err
	}

	ids := make(IDs, 0, len(matches))

	for _, m := range matches {
		base := filepath.Base(m)

		if base == "" {
			continue
		}
		id, err := ParseID(base)

		if err != nil {
			continue
		}

		ids = append(ids, id)
	}

	return ids, nil
}

func (ids IDs) Len() int {
	return len(ids)
}

func (ids IDs) Less(i, j int) bool {
	if len(ids[i]) < len(ids[j]) {
		return true
	}

	for k, b := range ids[i] {
		if b == ids[j][k] {
			continue
		}

		if b < ids[j][k] {
			return true
		} else {
			return false
		}
	}

	return false
}

func (ids IDs) Swap(i, j int) {
	ids[i], ids[j] = ids[j], ids[i]
}