frostfs-node/pkg/local_object_storage/bucket/fsbucket/treemethods_test.go

package fsbucket

import (
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"testing"

	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/bucket"
	"github.com/stretchr/testify/require"
	"go.uber.org/atomic"
)

func prepareTree(badFiles bool) (string, error) {
	name := make([]byte, 32)
	root, err := ioutil.TempDir("", "treeBucket_test")
	if err != nil {
		return "", err
	}

	// paths must contain strings with hex ascii symbols
	paths := [][]string{
		{root, "abcd"},
		{root, "abcd", "cdef"},
		{root, "abcd", "cd01"},
		{root, "0123", "2345"},
		{root, "0123", "2345", "4567"},
	}

	dirs := make([]string, len(paths))

	for i := range paths {
		dirs[i] = path.Join(paths[i]...)

		err = os.MkdirAll(dirs[i], 0700)
		if err != nil {
			return "", err
		}

		// create couple correct files
		for j := 0; j < 2; j++ {
			_, err := rand.Read(name)
			if err != nil {
				return "", err
			}

			filePrefix := new(strings.Builder)
			for k := 1; k < len(paths[i]); k++ {
				filePrefix.WriteString(paths[i][k])
			}
			filePrefix.WriteString(hex.EncodeToString(name))

			file, err := os.OpenFile(path.Join(dirs[i], filePrefix.String()), os.O_CREATE, 0700)
			if err != nil {
				return "", err
			}
			file.Close()
		}

		if !badFiles {
			continue
		}

		// create one bad file
		_, err := rand.Read(name)
		if err != nil {
			return "", err
		}

		file, err := os.OpenFile(path.Join(dirs[i], "fff"+hex.EncodeToString(name)), os.O_CREATE, 0700)
		if err != nil {
			return "", err
		}
		file.Close()
	}

	return root, nil
}

func TestTreebucket_List(t *testing.T) {
	root, err := prepareTree(true)
	require.NoError(t, err)
	defer os.RemoveAll(root)

	b := treeBucket{
		dir:          root,
		perm:         0700,
		depth:        1,
		prefixLength: 4,
	}
	results, err := b.List()
	require.NoError(t, err)
	require.Len(t, results, 2)

	b.depth = 2
	results, err = b.List()
	require.NoError(t, err)
	require.Len(t, results, 6)

	b.depth = 3
	results, err = b.List()
	require.NoError(t, err)
	require.Len(t, results, 2)

	b.depth = 4
	results, err = b.List()
	require.NoError(t, err)
	require.Len(t, results, 0)
}

func TestTreebucket(t *testing.T) {
	root, err := prepareTree(true)
	require.NoError(t, err)
	defer os.RemoveAll(root)

	b := treeBucket{
		dir:          root,
		perm:         0700,
		depth:        2,
		prefixLength: 4,
		sz:           atomic.NewInt64(0),
	}

	results, err := b.List()
	require.NoError(t, err)
	require.Len(t, results, 6)

	t.Run("Get", func(t *testing.T) {
		for i := range results {
			_, err = b.Get(results[i])
			require.NoError(t, err)
		}
		_, err = b.Get([]byte("Hello world!"))
		require.Error(t, err)
	})

	t.Run("Has", func(t *testing.T) {
		for i := range results {
			require.True(t, b.Has(results[i]))
		}
		require.False(t, b.Has([]byte("Unknown key")))
	})

	t.Run("Set", func(t *testing.T) {
		keyHash := sha256.Sum256([]byte("Set this key"))
		key := keyHash[:]
		value := make([]byte, 32)
		rand.Read(value)

		// set sha256 key
		err := b.Set(key, value)
		require.NoError(t, err)

		require.True(t, b.Has(key))
		data, err := b.Get(key)
		require.NoError(t, err)
		require.Equal(t, data, value)

		filename := hex.EncodeToString(key)
		_, err = os.Lstat(path.Join(root, filename[:4], filename[4:8], filename))
		require.NoError(t, err)

		// set key that cannot be placed in the required dir depth
		key, err = hex.DecodeString("abcdef")
		require.NoError(t, err)

		err = b.Set(key, value)
		require.Error(t, err)
	})

	t.Run("Delete", func(t *testing.T) {
		keyHash := sha256.Sum256([]byte("Delete this key"))
		key := keyHash[:]
		value := make([]byte, 32)
		rand.Read(value)

		err := b.Set(key, value)
		require.NoError(t, err)

		// delete sha256 key
		err = b.Del(key)
		require.NoError(t, err)

		_, err = b.Get(key)
		require.Error(t, err)
		filename := hex.EncodeToString(key)
		_, err = os.Lstat(path.Join(root, filename[:4], filename[4:8], filename))
		require.Error(t, err)
	})
}

func TestTreebucket_Close(t *testing.T) {
	root, err := prepareTree(true)
	require.NoError(t, err)
	defer os.RemoveAll(root)

	b := treeBucket{
		dir:          root,
		perm:         0700,
		depth:        2,
		prefixLength: 4,
	}
	err = b.Close()
	require.NoError(t, err)

	_, err = os.Lstat(root)
	require.Error(t, err)
}

func TestTreebucket_Size(t *testing.T) {
	root, err := prepareTree(true)
	require.NoError(t, err)
	defer os.RemoveAll(root)

	var size int64 = 1024
	key := []byte("Set this key")
	value := make([]byte, size)
	rand.Read(value)

	b := treeBucket{
		dir:          root,
		perm:         0700,
		depth:        2,
		prefixLength: 4,
		sz:           atomic.NewInt64(0),
	}

	err = b.Set(key, value)
	require.NoError(t, err)
	require.Equal(t, size, b.Size())
}

func BenchmarkTreebucket_List(b *testing.B) {
	root, err := prepareTree(false)
	defer os.RemoveAll(root)
	if err != nil {
		b.Error(err)
	}

	treeFSBucket := &treeBucket{
		dir:          root,
		perm:         0755,
		depth:        2,
		prefixLength: 4,
	}

	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, err := treeFSBucket.List()
		if err != nil {
			b.Error(err)
		}
	}
}

func BenchmarkFilewalkBucket_List(b *testing.B) {
	root, err := prepareTree(false)
	defer os.RemoveAll(root)
	if err != nil {
		b.Error(err)
	}

	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		buckets := make([]bucket.BucketItem, 0)

		filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
			if err != nil || info.IsDir() {
				return nil
			}

			val, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}

			key, err := decodeHexKey(info.Name())
			if err != nil {
				return err
			}

			buckets = append(buckets, bucket.BucketItem{
				Key: key,
				Val: val,
			})

			return nil
		})
	}
}

func BenchmarkTreeBucket_Size(b *testing.B) {
	root, err := prepareTree(false)
	defer os.RemoveAll(root)
	if err != nil {
		b.Error(err)
	}

	treeFSBucket := &treeBucket{
		dir:          root,
		perm:         0755,
		depth:        2,
		prefixLength: 4,
	}

	treeFSBucket.sz = atomic.NewInt64(treeFSBucket.size())

	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_ = treeFSBucket.Size()
	}
}