package khepri import ( "bytes" "encoding/json" "fmt" "io" "io/ioutil" "os" "os/user" "path/filepath" "strconv" "syscall" "time" "github.com/fd0/khepri/chunker" ) type Tree struct { Nodes []*Node `json:"nodes,omitempty"` } type Node struct { Name string `json:"name"` Type string `json:"type"` Mode os.FileMode `json:"mode,omitempty"` ModTime time.Time `json:"mtime,omitempty"` AccessTime time.Time `json:"atime,omitempty"` ChangeTime time.Time `json:"ctime,omitempty"` UID uint32 `json:"uid"` GID uint32 `json:"gid"` User string `json:"user,omitempty"` Group string `json:"group,omitempty"` Inode uint64 `json:"inode,omitempty"` Size uint64 `json:"size,omitempty"` Links uint64 `json:"links,omitempty"` LinkTarget string `json:"linktarget,omitempty"` Device uint64 `json:"device,omitempty"` Content []ID `json:"content,omitempty"` Subtree ID `json:"subtree,omitempty"` Tree *Tree `json:"-"` repo *Repository } func NewTree() *Tree { return &Tree{ Nodes: []*Node{}, } } func store_chunk(repo *Repository, rd io.Reader) (ID, error) { data, err := ioutil.ReadAll(rd) if err != nil { return nil, err } id, err := repo.Create(TYPE_BLOB, data) if err != nil { return nil, err } return id, nil } func NewTreeFromPath(repo *Repository, dir string) (*Tree, error) { fd, err := os.Open(dir) defer fd.Close() if err != nil { return nil, err } entries, err := fd.Readdir(-1) if err != nil { return nil, err } tree := &Tree{ Nodes: make([]*Node, 0, len(entries)), } for _, entry := range entries { path := filepath.Join(dir, entry.Name()) node, err := NodeFromFileInfo(path, entry) if err != nil { return nil, err } node.repo = repo tree.Nodes = append(tree.Nodes, node) if entry.IsDir() { node.Tree, err = NewTreeFromPath(repo, path) if err != nil { return nil, err } continue } if node.Type == "file" { file, err := os.Open(path) defer file.Close() if err != nil { return nil, err } if node.Size < chunker.MinSize { // if the file is small enough, store it directly id, err := store_chunk(repo, file) if err != nil { return nil, err } node.Content = []ID{id} } else { // else store chunks node.Content = []ID{} ch := chunker.New(file) for { chunk, err := ch.Next() if err == io.EOF { break } if err != nil { return nil, err } id, err := store_chunk(repo, bytes.NewBuffer(chunk.Data)) node.Content = append(node.Content, id) } } } } return tree, nil } func (tree *Tree) Save(repo *Repository) (ID, error) { for _, node := range tree.Nodes { if node.Tree != nil { var err error node.Subtree, err = node.Tree.Save(repo) if err != nil { return nil, err } } } data, err := json.Marshal(tree) if err != nil { return nil, err } id, err := repo.Create(TYPE_BLOB, data) if err != nil { return nil, err } return id, nil } func NewTreeFromRepo(repo *Repository, id ID) (*Tree, error) { tree := NewTree() rd, err := repo.Get(TYPE_BLOB, id) defer rd.Close() if err != nil { return nil, err } decoder := json.NewDecoder(rd) err = decoder.Decode(tree) if err != nil { return nil, err } for _, node := range tree.Nodes { node.repo = repo if node.Subtree != nil { node.Tree, err = NewTreeFromRepo(repo, node.Subtree) if err != nil { return nil, err } } } return tree, nil } func (tree *Tree) CreateAt(path string) error { for _, node := range tree.Nodes { nodepath := filepath.Join(path, node.Name) if node.Type == "dir" { err := os.Mkdir(nodepath, 0700) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) continue } err = os.Chmod(nodepath, node.Mode) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) continue } err = os.Chown(nodepath, int(node.UID), int(node.GID)) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) continue } err = node.Tree.CreateAt(filepath.Join(path, node.Name)) if err != nil { return err } err = os.Chtimes(nodepath, node.AccessTime, node.ModTime) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) continue } } else { err := node.CreateAt(nodepath) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) continue } } } return nil } // TODO: make sure that node.Type is valid func (node *Node) fill_extra(path string, fi os.FileInfo) (err error) { stat, ok := fi.Sys().(*syscall.Stat_t) if !ok { return } node.ChangeTime = time.Unix(stat.Ctim.Unix()) node.AccessTime = time.Unix(stat.Atim.Unix()) node.UID = stat.Uid node.GID = stat.Gid if u, nil := user.LookupId(strconv.Itoa(int(stat.Uid))); err == nil { node.User = u.Username } // TODO: implement getgrnam() // if g, nil := user.LookupId(strconv.Itoa(int(stat.Uid))); err == nil { // node.User = u.Username // } node.Inode = stat.Ino switch node.Type { case "file": node.Size = uint64(stat.Size) node.Links = stat.Nlink case "dir": // nothing to do case "symlink": node.LinkTarget, err = os.Readlink(path) case "dev": node.Device = stat.Rdev case "chardev": node.Device = stat.Rdev case "fifo": // nothing to do case "socket": // nothing to do default: panic(fmt.Sprintf("invalid node type %q", node.Type)) } return err } func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { node := &Node{ Name: fi.Name(), Mode: fi.Mode() & os.ModePerm, ModTime: fi.ModTime(), } switch fi.Mode() & (os.ModeType | os.ModeCharDevice) { case 0: node.Type = "file" case os.ModeDir: node.Type = "dir" case os.ModeSymlink: node.Type = "symlink" case os.ModeDevice | os.ModeCharDevice: node.Type = "chardev" case os.ModeDevice: node.Type = "dev" case os.ModeNamedPipe: node.Type = "fifo" case os.ModeSocket: node.Type = "socket" } err := node.fill_extra(path, fi) return node, err } func (node *Node) CreateAt(path string) error { if node.repo == nil { return fmt.Errorf("repository is nil!") } switch node.Type { case "file": // TODO: handle hard links f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600) defer f.Close() if err != nil { return err } for _, blobid := range node.Content { rd, err := node.repo.Get(TYPE_BLOB, blobid) if err != nil { return err } _, err = io.Copy(f, rd) if err != nil { return err } } f.Close() case "symlink": err := os.Symlink(node.LinkTarget, path) if err != nil { return err } err = os.Lchown(path, int(node.UID), int(node.GID)) if err != nil { return err } f, err := os.OpenFile(path, O_PATH|syscall.O_NOFOLLOW, 0600) defer f.Close() if err != nil { return err } var utimes = []syscall.Timeval{ syscall.NsecToTimeval(node.AccessTime.UnixNano()), syscall.NsecToTimeval(node.ModTime.UnixNano()), } err = syscall.Futimes(int(f.Fd()), utimes) if err != nil { return err } return nil case "dev": err := syscall.Mknod(path, syscall.S_IFBLK|0600, int(node.Device)) if err != nil { return err } case "chardev": err := syscall.Mknod(path, syscall.S_IFCHR|0600, int(node.Device)) if err != nil { return err } case "fifo": err := syscall.Mkfifo(path, 0600) if err != nil { return err } case "socket": // nothing to do, we do not restore sockets default: return fmt.Errorf("filetype %q not implemented!\n", node.Type) } err := os.Chmod(path, node.Mode) if err != nil { return err } err = os.Chown(path, int(node.UID), int(node.GID)) if err != nil { return err } err = os.Chtimes(path, node.AccessTime, node.ModTime) if err != nil { return err } return nil }