frostfs-node/pkg/local_object_storage/writecache/writecachebitcask/writecachebitcask.go

package writecachebitcask

import (
	"sync/atomic"
	"time"

	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
	"go.uber.org/zap"
)

/*

The cache operates as a hash table where the key space is split in regions, each with its own lock (a sync.Mutex).
Each region maintains a key directory and appends updates (PUTs and DELETEs) to a log file. A key directory
entry stores the address, the log file index and the offset within the logfile. This structure is similar to
a bitcask (see https://en.wikipedia.org/wiki/Bitcask and https://riak.com/assets/bitcask-intro.pdf).

Since the file writes use O_SYNC, the updates are batched in a memory buffer first, so that it incurs fewer writes
under heavy load. After a log file reaches maximum capacity, it's closed and a new one is started. The completed log
files are pushed to the flushing process which processes them one by one and deletes them after updating the underlying
storage. The flushing process doesn't poll periodically for new work; instead, there's a buffered channel to which the
log file indices are pushed. If this channel fills, it will block the sending process (a request) until there's room
to continue, providing backpressure to the client when the underlying storage is not able to keep up.

The lower bytes of the object addresses are used as region and bucket hashes, since they are already hashed values
by construction.

                                           │               │
                        Regions            │   In-Memory   │ In-Disk
                    ───────────────────────┼───────────────┼─────────────────────────────────────────────────────────────────────────────────
         ┌─          ┌─                    │               │ ┌────────────────────────────────────────┐ ┌───────────┐ ┌───────────┐
         │           │  Bucket 1           │    KeyDir     │ │                 LogFile 0              │ │ LogFile 1 │ │ LogFile 2 │
         │  Region 1 │  Bucket 2           │ ┌───────────┐ │ │┌──────────────┐ ┌──────────────┐       │ │           │ │           │
         │           │    (...)            │ │ MemBuffer │ │ ││Addr Size Data│ │Addr Size Data│ (...) │ │   (...)   │ │   (...)   │  (...)
         │   (Mutex) │  Bucket K           │ └───────────┘ │ │└──────────────┘ └──────────────┘       │ │           │ │           │
         │           └─                    │               │ └────────────────────────────────────────┘ └───────────┘ └───────────┘
         │          ───────────────────────┼───────────────┼─────────────────────────────────────────────────────────────────────────────────
         │           ┌─                    │               │ ┌────────────────────────────────────────┐ ┌───────────┐ ┌───────────┐
         │           │  Bucket K+1         │    KeyDir     │ │                 LogFile 0              │ │ LogFile 1 │ │ LogFile 2 │
         │  Region 2 │  Bucket K+2         │ ┌───────────┐ │ │┌──────────────┐ ┌──────────────┐       │ │           │ │           │
         │           │    (...)            │ │ MemBuffer │ │ ││Addr Size Data│ │Addr Size Data│ (...) │ │   (...)   │ │   (...)   │  (...)
  Key    │           │  Bucket 2K          │ └───────────┘ │ │└──────────────┘ └──────────────┘       │ │           │ │           │
 Space   │           └─                    │               │ └────────────────────────────────────────┘ └───────────┘ └───────────┘
         │          ───────────────────────┼───────────────┼─────────────────────────────────────────────────────────────────────────────────
         │                   (...)         │     (...)     │                    (...)
         │          ───────────────────────┼───────────────┼─────────────────────────────────────────────────────────────────────────────────
         │           ┌─                    │               │ ┌────────────────────────────────────────┐ ┌───────────┐ ┌───────────┐
         │           │  Bucket (N-1)*K+1   │    KeyDir     │ │                 LogFile 0              │ │ LogFile 1 │ │ LogFile 2 │
         │  Region N │  Bucket (N-1)*K+2   │ ┌───────────┐ │ │┌──────────────┐ ┌──────────────┐       │ │           │ │           │
         │           │    (...)            │ │ MemBuffer │ │ ││Addr Size Data│ │Addr Size Data│ (...) │ │   (...)   │ │   (...)   │  (...)
         │           │  Bucket N*K         │ └───────────┘ │ │└──────────────┘ └──────────────┘       │ │           │ │           │
         └─          └─                    │               │ └────────────────────────────────────────┘ └───────────┘ └───────────┘
                    ───────────────────────┴───────────────┴─────────────────────────────────────────────────────────────────────────────────

*/

type cache struct {
	options

	mode    atomic.Uint32
	closed  atomic.Bool
	regions []*region
}

func New(opts ...Option) writecache.Cache {
	c := &cache{
		options: options{
			log:     &logger.Logger{Logger: zap.NewNop()},
			metrics: writecache.DefaultMetrics(),

			maxObjectSize:          128 << 10,
			bucketCount:            1 << 16,
			regionCount:            1 << 2,
			maxLogSize:             64 << 20,
			maxBatchDelay:          1 * time.Millisecond,
			maxPendingLogFileFlush: 4,
		},
	}
	for i := range opts {
		opts[i](&c.options)
	}
	return c
}