From f88e0866fed5b2b325a3661b09f8e449435eb4b1 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Mon, 8 Feb 2021 21:25:17 +0300 Subject: [PATCH] [#316] locode: Implement NeoFS location database based on BoltDB instance Define NeoFS location database based on BoltDB. Implement methods to save the record by key (Put) and to read the record by key (Get). Signed-off-by: Leonard Lyubich --- pkg/util/locode/db/boltdb/calls.go | 162 +++++++++++++++++++++++++++++ pkg/util/locode/db/boltdb/db.go | 73 +++++++++++++ pkg/util/locode/db/boltdb/opts.go | 22 ++++ 3 files changed, 257 insertions(+) create mode 100644 pkg/util/locode/db/boltdb/calls.go create mode 100644 pkg/util/locode/db/boltdb/db.go create mode 100644 pkg/util/locode/db/boltdb/opts.go diff --git a/pkg/util/locode/db/boltdb/calls.go b/pkg/util/locode/db/boltdb/calls.go new file mode 100644 index 00000000..2aca8560 --- /dev/null +++ b/pkg/util/locode/db/boltdb/calls.go @@ -0,0 +1,162 @@ +package locodebolt + +import ( + "encoding/json" + "os" + "path" + + locodedb "github.com/nspcc-dev/neofs-node/pkg/util/locode/db" + "github.com/pkg/errors" + "go.etcd.io/bbolt" +) + +// Open opens underlying BoltDB instance. +func (db *DB) Open() error { + // copy-paste from metabase: + // consider universal Open/Close for BoltDB wrappers + + err := os.MkdirAll(path.Dir(db.path), db.mode) + if err != nil { + return errors.Wrap(err, "could not create dir for BoltDB") + } + + db.bolt, err = bbolt.Open(db.path, db.mode, db.boltOpts) + if err != nil { + return errors.Wrap(err, "could not open BoltDB") + } + + return nil +} + +// Close closes underlying BoltDB instance. +// +// Must not be called before successful Open call. +func (db *DB) Close() error { + return db.bolt.Close() +} + +func countryBucketKey(cc *locodedb.CountryCode) ([]byte, error) { + return []byte(cc.String()), nil +} + +func locationBucketKey(lc *locodedb.LocationCode) ([]byte, error) { + return []byte(lc.String()), nil +} + +type recordJSON struct { + CountryName string + CityName string + SubDivName string + SubDivCode string + Latitude float64 + Longitude float64 + Continent string +} + +func recordValue(r locodedb.Record) ([]byte, error) { + p := r.GeoPoint() + + rj := &recordJSON{ + CountryName: r.CountryName(), + CityName: r.CityName(), + SubDivName: r.SubDivName(), + SubDivCode: r.SubDivCode(), + Latitude: p.Latitude(), + Longitude: p.Longitude(), + Continent: r.Continent().String(), + } + + return json.Marshal(rj) +} + +func recordFromValue(data []byte) (*locodedb.Record, error) { + rj := new(recordJSON) + + if err := json.Unmarshal(data, rj); err != nil { + return nil, err + } + + r := new(locodedb.Record) + r.SetCountryName(rj.CountryName) + r.SetCityName(rj.CityName) + r.SetSubDivName(rj.SubDivName) + r.SetSubDivCode(rj.SubDivCode) + r.SetGeoPoint(locodedb.NewPoint(rj.Latitude, rj.Longitude)) + + cont := locodedb.ContinentFromString(rj.Continent) + r.SetContinent(&cont) + + return r, nil +} + +// Put saves the record by key in underlying BoltDB instance. +// +// Country code from key is used for allocating the 1st level buckets. +// Records are stored in country buckets by location code from key. +// The records are stored in internal binary JSON format. +// +// Must not be called before successful Open call. +func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error { + return db.bolt.Update(func(tx *bbolt.Tx) error { + countryKey, err := countryBucketKey(key.CountryCode()) + if err != nil { + return err + } + + bktCountry, err := tx.CreateBucketIfNotExists(countryKey) + if err != nil { + return errors.Wrap(err, "could not create country bucket") + } + + // TODO: write country name once in Country bucket + + locationKey, err := locationBucketKey(key.LocationCode()) + if err != nil { + return err + } + + cont, err := recordValue(rec) + if err != nil { + return err + } + + return bktCountry.Put(locationKey, cont) + }) +} + +var errRecordNotFound = errors.New("record not found") + +// Put reads the record by key from underlying BoltDB instance. +// +// Returns an error if no record is presented by key in DB. +// +// Must not be called before successful Open call. +func (db *DB) Get(key locodedb.Key) (rec *locodedb.Record, err error) { + err = db.bolt.View(func(tx *bbolt.Tx) error { + countryKey, err := countryBucketKey(key.CountryCode()) + if err != nil { + return err + } + + bktCountry := tx.Bucket(countryKey) + if bktCountry == nil { + return errRecordNotFound + } + + locationKey, err := locationBucketKey(key.LocationCode()) + if err != nil { + return err + } + + data := bktCountry.Get(locationKey) + if data == nil { + return errRecordNotFound + } + + rec, err = recordFromValue(data) + + return err + }) + + return +} diff --git a/pkg/util/locode/db/boltdb/db.go b/pkg/util/locode/db/boltdb/db.go new file mode 100644 index 00000000..53a23b22 --- /dev/null +++ b/pkg/util/locode/db/boltdb/db.go @@ -0,0 +1,73 @@ +package locodebolt + +import ( + "fmt" + "os" + + "go.etcd.io/bbolt" +) + +// Prm groups the required parameters of the DB's constructor. +// +// All values must comply with the requirements imposed on them. +// Passing incorrect parameter values will result in constructor +// failure (error or panic depending on the implementation). +type Prm struct { + // Path to BoltDB file with NeoFS location database. + // + // Must not be empty. + Path string +} + +// DB is a descriptor of the NeoFS BoltDB location database. +// +// For correct operation, DB must be created +// using the constructor (New) based on the required parameters +// and optional components. +// +// After successful creation, +// DB must be opened through Open call. After successful opening, +// DB is ready to work through API (until Close call). +// +// Upon completion of work with the DB, it must be closed +// by Close method. +type DB struct { + path string + + mode os.FileMode + + boltOpts *bbolt.Options + + bolt *bbolt.DB +} + +const invalidPrmValFmt = "invalid parameter %s (%T):%v" + +func panicOnPrmValue(n string, v interface{}) { + panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) +} + +// New creates a new instance of the DB. +// +// Panics if at least one value of the parameters is invalid. +// +// The created DB requires calling the Open method in order +// to initialize required resources. +func New(prm Prm, opts ...Option) *DB { + switch { + case prm.Path == "": + panicOnPrmValue("Path", prm.Path) + } + + o := defaultOpts() + + for i := range opts { + opts[i](o) + } + + return &DB{ + path: prm.Path, + mode: o.mode, + boltOpts: o.boltOpts, + } +} diff --git a/pkg/util/locode/db/boltdb/opts.go b/pkg/util/locode/db/boltdb/opts.go new file mode 100644 index 00000000..e9eb9017 --- /dev/null +++ b/pkg/util/locode/db/boltdb/opts.go @@ -0,0 +1,22 @@ +package locodebolt + +import ( + "os" + + "go.etcd.io/bbolt" +) + +// Option sets an optional parameter of DB. +type Option func(*options) + +type options struct { + mode os.FileMode + + boltOpts *bbolt.Options +} + +func defaultOpts() *options { + return &options{ + mode: os.ModePerm, // 0777 + } +}