TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 101 Pull requests 5 Projects Releases 1 Activity Actions

Add Inner Ring code

Browse source
This commit is contained in:
Stanislav Bogatyrev 2020-07-24 16:54:03 +03:00
parent dadfd90dcd
commit b7b5079934
400 changed files with 11326 additions and 8595 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

129
.github/logo.svg vendored Normal file
View file

@ -0,0 +1,129 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
sodipodi:docname="logo_fs.svg"
inkscape:version="1.0 (4035a4fb49, 2020-05-01)"
id="svg57"
version="1.1"
viewBox="0 0 105 25"
height="25mm"
width="105mm">
<defs
id="defs51">
<clipPath
clipPathUnits="userSpaceOnUse"
id="clipPath434">
<path
d="M 0,0 H 1366 V 768 H 0 Z"
id="path432" />
</clipPath>
</defs>
<sodipodi:namedview
inkscape:window-maximized="0"
inkscape:window-y="0"
inkscape:window-x="130"
inkscape:window-height="1040"
inkscape:window-width="1274"
height="50mm"
units="mm"
showgrid="false"
inkscape:document-rotation="0"
inkscape:current-layer="layer1"
inkscape:document-units="mm"
inkscape:cy="344.49897"
inkscape:cx="468.64708"
inkscape:zoom="0.7"
inkscape:pageshadow="2"
inkscape:pageopacity="0.0"
borderopacity="1.0"
bordercolor="#666666"
pagecolor="#ffffff"
id="base" />
<metadata
id="metadata54">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<g
id="layer1"
inkscape:groupmode="layer"
inkscape:label="Layer 1">
<g
id="g424"
transform="matrix(0.35277777,0,0,-0.35277777,63.946468,10.194047)">
<path
d="m 0,0 v -8.093 h 12.287 v -3.94 H 0 V -24.067 H -4.534 V 3.898 H 15.677 V 0 Z"
style="fill:#00e396;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path426" />
</g>
<g
transform="matrix(0.35277777,0,0,-0.35277777,-315.43002,107.34005)"
id="g428">
<g
id="g430"
clip-path="url(#clipPath434)">
<g
id="g436"
transform="translate(1112.874,278.2981)">
<path
d="M 0,0 C 1.822,-0.932 3.354,-2.359 4.597,-4.28 L 1.165,-7.373 c -0.791,1.695 -1.779,2.924 -2.966,3.686 -1.186,0.763 -2.768,1.145 -4.745,1.145 -1.949,0 -3.461,-0.389 -4.534,-1.166 -1.074,-0.777 -1.61,-1.772 -1.61,-2.987 0,-1.13 0.523,-2.027 1.568,-2.69 1.045,-0.664 2.909,-1.236 5.593,-1.716 2.514,-0.452 4.512,-1.024 5.995,-1.716 1.483,-0.693 2.564,-1.554 3.242,-2.585 0.677,-1.031 1.016,-2.309 1.016,-3.834 0,-1.639 -0.466,-3.079 -1.398,-4.322 -0.932,-1.243 -2.239,-2.197 -3.919,-2.86 -1.681,-0.664 -3.623,-0.996 -5.826,-0.996 -5.678,0 -9.689,1.892 -12.033,5.678 l 3.178,3.178 c 0.903,-1.695 2.068,-2.939 3.495,-3.729 1.426,-0.791 3.199,-1.186 5.318,-1.186 2.005,0 3.58,0.345 4.724,1.038 1.144,0.692 1.716,1.674 1.716,2.945 0,1.017 -0.516,1.835 -1.547,2.457 -1.031,0.621 -2.832,1.172 -5.402,1.653 -2.571,0.479 -4.618,1.073 -6.143,1.779 -1.526,0.706 -2.635,1.582 -3.326,2.627 -0.693,1.045 -1.039,2.316 -1.039,3.813 0,1.582 0.438,3.023 1.314,4.322 0.875,1.299 2.14,2.33 3.792,3.093 1.653,0.763 3.58,1.144 5.783,1.144 C -4.018,1.398 -1.822,0.932 0,0"
style="fill:#00e396;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path438" />
</g>
<g
id="g440"
transform="translate(993.0239,277.5454)">
<path
d="m 0,0 c 2.054,-1.831 3.083,-4.465 3.083,-7.902 v -17.935 h -4.484 v 16.366 c 0,2.914 -0.626,5.024 -1.877,6.332 -1.253,1.308 -2.924,1.962 -5.016,1.962 -1.495,0 -2.896,-0.327 -4.204,-0.981 -1.308,-0.654 -2.381,-1.719 -3.222,-3.194 -0.841,-1.477 -1.261,-3.335 -1.261,-5.576 v -14.909 h -4.484 V 1.328 l 4.086,-1.674 0.118,-1.84 c 0.933,1.681 2.222,2.923 3.867,3.727 1.643,0.803 3.493,1.205 5.548,1.205 C -4.671,2.746 -2.055,1.83 0,0"
style="fill:#000033;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path442" />
</g>
<g
id="g444"
transform="translate(1027.9968,264.0386)">
<path
d="m 0,0 h -21.128 c 0.261,-2.84 1.205,-5.044 2.83,-6.613 1.625,-1.57 3.727,-2.355 6.305,-2.355 2.054,0 3.763,0.356 5.128,1.065 1.363,0.71 2.288,1.738 2.774,3.083 l 3.755,-1.961 c -1.121,-1.981 -2.616,-3.495 -4.484,-4.54 -1.868,-1.046 -4.259,-1.569 -7.173,-1.569 -4.223,0 -7.538,1.289 -9.948,3.867 -2.41,2.578 -3.615,6.146 -3.615,10.704 0,4.558 1.149,8.127 3.447,10.705 2.298,2.578 5.557,3.867 9.779,3.867 2.615,0 4.876,-0.58 6.782,-1.738 1.905,-1.158 3.343,-2.728 4.315,-4.707 C -0.262,7.827 0.224,5.605 0.224,3.139 0.224,2.092 0.149,1.046 0,0 m -18.298,10.144 c -1.513,-1.457 -2.438,-3.512 -2.775,-6.165 h 16.982 c -0.3,2.615 -1.159,4.661 -2.578,6.137 -1.42,1.476 -3.307,2.214 -5.661,2.214 -2.466,0 -4.455,-0.728 -5.968,-2.186"
style="fill:#000033;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path446" />
</g>
<g
id="g448"
transform="translate(1057.8818,276.4246)">
<path
d="m 0,0 c 2.41,-2.578 3.615,-6.147 3.615,-10.705 0,-4.558 -1.205,-8.126 -3.615,-10.704 -2.41,-2.578 -5.726,-3.867 -9.948,-3.867 -4.222,0 -7.537,1.289 -9.947,3.867 -2.41,2.578 -3.615,6.146 -3.615,10.704 0,4.558 1.205,8.127 3.615,10.705 2.41,2.578 5.725,3.867 9.947,3.867 C -5.726,3.867 -2.41,2.578 0,0 m -16.617,-2.858 c -1.607,-1.906 -2.41,-4.522 -2.41,-7.847 0,-3.326 0.803,-5.94 2.41,-7.846 1.607,-1.905 3.83,-2.858 6.669,-2.858 2.839,0 5.063,0.953 6.67,2.858 1.606,1.906 2.41,4.52 2.41,7.846 0,3.325 -0.804,5.941 -2.41,7.847 C -4.885,-0.953 -7.109,0 -9.948,0 c -2.839,0 -5.062,-0.953 -6.669,-2.858"
style="fill:#000033;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path450" />
</g>
</g>
</g>
<g
id="g452"
transform="matrix(0.35277777,0,0,-0.35277777,5.8329581,6.5590171)">
<path
d="m 0,0 0.001,-38.946 25.286,-9.076 V -8.753 L 52.626,1.321 27.815,10.207 Z"
style="fill:#00e599;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path454" />
</g>
<g
id="g456"
transform="matrix(0.35277777,0,0,-0.35277777,15.479008,10.041927)">
<path
d="M 0,0 V -21.306 L 25.293,-30.364 25.282,9.347 Z"
style="fill:#00b091;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path458" />
</g>
</g>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 6.5 KiB

1
.gitignore vendored
View file

@ -1,3 +1,4 @@
.idea
bin
temp
cmd/test

14
CHANGELOG.md
View file

@ -1,6 +1,20 @@
# Changelog
Changelog for NeoFS Node
## [0.11.0] - 2020-07-23
### Added
- Inner ring application to repository.
- Inner ring epoch processor.
- Inner ring asset processor for GAS deposit and withdraw.
### Changed
- The structure of source code tree.
## [0.10.0] - 2020-07-10
First public review release.
[0.11.0]: https://github.com/nspcc-dev/neofs-api-go/compare/v0.10.0...v0.11.0

60
CONTRIBUTING.md
View file

@ -1,3 +1,61 @@
# Contributing
We do not accept any contributions. As yet.
## DCO Sign off
All authors to the project retain copyright to their work. However, to ensure
that they are only submitting work that they have rights to, we are requiring
everyone to acknowledge this by signing their work.
Any copyright notices in this repository should specify the authors as "the
contributors".
To sign your work, just add a line like this at the end of your commit message:
```
Signed-off-by: Samii Sakisaka <samii@nspcc.ru>
```
This can easily be done with the `--signoff` option to `git commit`.
By doing this you state that you can certify the following (from [The Developer
Certificate of Origin](https://developercertificate.org/):
```
Developer Certificate of Origin
Version 1.1
Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
1 Letterman Drive
Suite D4700
San Francisco, CA, 94129
Everyone is permitted to copy and distribute verbatim copies of this
license document, but changing it is not allowed.
Developer's Certificate of Origin 1.1
By making a contribution to this project, I certify that:
(a) The contribution was created in whole or in part by me and I
have the right to submit it under the open source license
indicated in the file; or
(b) The contribution is based upon previous work that, to the best
of my knowledge, is covered under an appropriate open source
license and I have the right under that license to submit that
work with modifications, whether created in whole or in part
by me, under the same open source license (unless I am
permitted to submit under a different license), as indicated
in the file; or
(c) The contribution was provided directly to me by some other
person who certified (a), (b) or (c) and I have not modified
it.
(d) I understand and agree that this project and the contribution
are public and that a record of the contribution (including all
personal information I submit with it, including my sign-off) is
maintained indefinitely and may be redistributed consistent with
this project or the open source license(s) involved.
```

26
CREDITS.md Normal file
View file

@ -0,0 +1,26 @@
# Credits
Initial NeoFS research and development (2018-2020) was done by
[NeoSPCC](https://nspcc.ru) team.
In alphabetical order:
- Alexey Vanin
- Anastasia Prasolova
- Anatoly Bogatyrev
- Evgeny Kulikov
- Evgeny Stratonikov
- Leonard Liubich
- Sergei Liubich
- Stanislav Bogatyrev
# Contributors
In chronological order:
# Special Thanks
For product development support:
- Fabian Wahle
- Neo Global Development

7
Dockerfile
View file

@ -1,14 +1,13 @@
FROM golang:1.14-alpine as builder
FROM golang:1.14-alpine as basebuilder
RUN apk add --update make bash
FROM basebuilder as builder
ARG BUILD=now
ARG VERSION=dev
ARG REPO=repository
WORKDIR /src
COPY . /src
RUN apk add --update make bash
RUN make bin/neofs-node
# Executable image

20
Dockerfile.ir Normal file
View file

@ -0,0 +1,20 @@
FROM golang:1.14-alpine as basebuilder
RUN apk add --update make bash
FROM basebuilder as builder
ARG BUILD=now
ARG VERSION=dev
ARG REPO=repository
WORKDIR /src
COPY . /src
RUN make bin/neofs-ir
# Executable image
FROM scratch AS neofs-ir
WORKDIR /
COPY --from=builder /src/bin/neofs-ir /bin/neofs-ir
CMD ["neofs-ir"]

26
Makefile
View file

@ -11,16 +11,15 @@ BIN = bin
DIRS= $(BIN)
# List of binaries to build. May be automated.
CMDS = neofs-node
CMDS = neofs-node neofs-ir
CMS = $(addprefix $(BIN)/, $(CMDS))
BINS = $(addprefix $(BIN)/, $(CMDS))
.PHONY: help dep clean fmt
# To build a specific binary, use it's name prfixed with bin/ as a target
# For example `make bin/neofs-node` will buils only Storage node binary
# Just `make` will
# Build all possible binaries
# To build a specific binary, use it's name prefix with bin/ as a target
# For example `make bin/neofs-node` will build only storage node binary
# Just `make` will build all possible binaries
all: $(DIRS) $(BINS)
$(BINS): $(DIRS) dep
@ -41,7 +40,7 @@ dep:
@go mod tidy -v && echo OK || (echo fail && exit 2)
@printf "⇒ Download requirements: "
@go mod download && echo OK || (echo fail && exit 2)
@printf "⇒ Store vendor localy: "
@printf "⇒ Store vendor locally: "
@go mod vendor && echo OK || (echo fail && exit 2)
# Regenerate proto files:
@ -60,17 +59,26 @@ protoc:
--gofast_out=plugins=grpc,paths=source_relative:. $$f; \
done
# Build NeoFS Sorage Node docker image
# Build NeoFS Storage Node docker image
image-storage:
@echo "⇒ Build NeoFS Sorage Node docker image "
@echo "⇒ Build NeoFS Storage Node docker image "
@docker build \
--build-arg REPO=$(REPO) \
--build-arg VERSION=$(VERSION) \
-f Dockerfile \
-t $(HUB_IMAGE)-storage:$(HUB_TAG) .
# Build NeoFS Storage Node docker image
image-ir:
@echo "⇒ Build NeoFS Inner Ring docker image "
@docker build \
--build-arg REPO=$(REPO) \
--build-arg VERSION=$(VERSION) \
-f Dockerfile.ir \
-t $(HUB_IMAGE)-ir:$(HUB_TAG) .
# Build all Docker images
images: image-storage
images: image-storage image-ir
# Reformat code
fmt:

57
README.md Normal file
View file

@ -0,0 +1,57 @@
<p align="center">
<img src="./.github/logo.svg" width="500px" alt="NeoFS">
</p>
<p align="center">
<a href="https://fs.neo.org">NeoFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
</p>
---
[![Report](https://goreportcard.com/badge/github.com/nspcc-dev/neo-go)](https://goreportcard.com/report/github.com/nspcc-dev/neofs-node)
![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/nspcc-dev/neofs-node?sort=semver)
![License](https://img.shields.io/github/license/nspcc-dev/neofs-node.svg?style=popout)
# Overview
NeoFS Nodes are organized in peer-to-peer network that takes care of storing and
distributing user's data. Any Neo user may participate in the network and get
paid for providing storage resources to other users or store his data in NeoFS
and pay a competitive price for it.
Users can reliably store object data in the NeoFS network and have a transparent
data placement process due to decentralized architecture and flexible storage
policies. Each node is responsible for executing the storage policies that the
users select for geographical location, reliability level, number of nodes, type
of disks, capacity, etc. Thus, NeoFS gives full control over data to users.
Deep [Neo Blockchain](https://neo.org) integration allows NeoFS to be used by
dApp directly from
[NeoVM](https://docs.neo.org/docs/en-us/basic/technology/neovm.html) on the
[Smart Contract](https://docs.neo.org/docs/en-us/basic/technology/neocontract.html)
code level. This way dApps are not limited to on-chain storage and can
manipulate large amounts of data without paying a prohibitive price.
NeoFS has native [gRPC](https://grpc.io) API and popular protocol gates such as
[AWS S3](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html),
[HTTP](https://wikipedia.org/wiki/Hypertext_Transfer_Protocol),
[FUSE](https://wikipedia.org/wiki/Filesystem_in_Userspace) and
[sFTP](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol) allowing
developers to easily integrate applications without rewriting their code.
# Contributing
Feel free to contribute to this project after reading the [contributing
guidelines](CONTRIBUTING.md).
Before starting to work on a certain topic, create an new issue first,
describing the feature/topic you are going to implement.
# Credits
NeoFS is maintained by [NeoSPCC](https://nspcc.ru) with the help and
contributions from community members.
Please see [CREDITS](CREDITS.md) for details.
# License
- [GNU General Public License v3.0](LICENSE)

74
cmd/neofs-ir/defaults.go Normal file
View file

@ -0,0 +1,74 @@
package main
import (
"strings"
"github.com/nspcc-dev/neo-go/pkg/config/netmode"
"github.com/nspcc-dev/neofs-node/misc"
"github.com/spf13/viper"
)
func newConfig(path string) (*viper.Viper, error) {
var (
err error
v = viper.New()
)
v.SetEnvPrefix(misc.InnerRingPrefix)
v.AutomaticEnv()
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
v.SetDefault("app.name", misc.InnerRingName)
v.SetDefault("app.version", misc.Version)
defaultConfiguration(v)
if path != "" {
v.SetConfigFile(path)
v.SetConfigType("yml") // fixme: for now
err = v.ReadInConfig()
}
return v, err
}
func defaultConfiguration(cfg *viper.Viper) {
cfg.SetDefault("logger.level", "info")
cfg.SetDefault("logger.format", "console")
cfg.SetDefault("logger.trace_level", "fatal")
cfg.SetDefault("logger.no_disclaimer", false)
cfg.SetDefault("logger.sampling.initial", 1000)
cfg.SetDefault("logger.sampling.thereafter", 1000)
cfg.SetDefault("pprof.enabled", false)
cfg.SetDefault("pprof.address", ":6060")
cfg.SetDefault("pprof.shutdown_ttl", "30s")
cfg.SetDefault("metrics.enabled", false)
cfg.SetDefault("metrics.address", ":9090")
cfg.SetDefault("metrics.shutdown_ttl", "30s")
cfg.SetDefault("morph.endpoint.client", "")
cfg.SetDefault("morph.endpoint.notification", "")
cfg.SetDefault("morph.dial_timeout", "10s")
cfg.SetDefault("morph.magic_number", uint32(netmode.PrivNet))
cfg.SetDefault("mainnet.endpoint.client", "")
cfg.SetDefault("mainnet.endpoint.notification", "")
cfg.SetDefault("mainnet.dial_timeout", "10s")
cfg.SetDefault("mainnet.magic_number", uint32(netmode.PrivNet))
cfg.SetDefault("key", "") // inner ring node key
cfg.SetDefault("contracts.netmap", "")
cfg.SetDefault("contracts.neofs", "")
cfg.SetDefault("contracts.balance", "")
// gas native contract
cfg.SetDefault("contracts.gas", "8c23f196d8a1bfd103a9dcb1f9ccf0c611377d3b")
cfg.SetDefault("timers.epoch", "5s")
cfg.SetDefault("workers.netmap", "10")
cfg.SetDefault("workers.balance", "10")
cfg.SetDefault("workers.neofs", "10")
}

82
cmd/neofs-ir/main.go Normal file
View file

@ -0,0 +1,82 @@
package main
import (
"flag"
"fmt"
"os"
"github.com/nspcc-dev/neofs-node/misc"
"github.com/nspcc-dev/neofs-node/pkg/innerring"
"github.com/nspcc-dev/neofs-node/pkg/util/grace"
"github.com/nspcc-dev/neofs-node/pkg/util/logger"
"github.com/nspcc-dev/neofs-node/pkg/util/profiler"
)
const (
// ErrorReturnCode returns when application crashed at initialization stage
ErrorReturnCode = 1
// SuccessReturnCode returns when application closed without panic
SuccessReturnCode = 0
)
func exitErr(err error) {
if err != nil {
fmt.Println(err)
os.Exit(ErrorReturnCode)
}
}
func main() {
configFile := flag.String("config", "", "path to config")
versionFlag := flag.Bool("version", false, "neofs-ir node version")
flag.Parse()
if *versionFlag {
fmt.Println("version:", misc.Version)
os.Exit(SuccessReturnCode)
}
cfg, err := newConfig(*configFile)
exitErr(err)
log, err := logger.NewLogger(cfg)
exitErr(err)
ctx := grace.NewGracefulContext(log)
pprof := profiler.NewProfiler(log, cfg)
prometheus := profiler.NewMetrics(log, cfg)
innerRing, err := innerring.New(ctx, log, cfg)
if err != nil {
exitErr(err)
}
// start pprof if enabled
if pprof != nil {
pprof.Start(ctx)
defer pprof.Stop()
}
// start prometheus if enabled
if prometheus != nil {
prometheus.Start(ctx)
defer prometheus.Stop()
}
// start inner ring
err = innerRing.Start(ctx)
if err != nil {
exitErr(err)
}
log.Info("application started")
// todo: select ctx.Done or exported error channel
<-ctx.Done()
innerRing.Stop()
log.Info("application stopped")
}

34
cmd/neofs-node/defaults.go
View file

@ -4,8 +4,7 @@ import (
"time"
"github.com/nspcc-dev/neo-go/pkg/config/netmode"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/nspcc-dev/neofs-node/modules/morph"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/morph"
"github.com/spf13/viper"
)
@ -65,23 +64,6 @@ func setDefaults(v *viper.Viper) {
})
}
// Storage section
{
storageTypes := []string{
core.BlobStore.String(),
core.MetaStore.String(),
core.SpaceMetricsStore.String(),
}
for i := range storageTypes {
v.SetDefault("storage."+storageTypes[i]+".bucket", "boltdb")
v.SetDefault("storage."+storageTypes[i]+".path", "./temp/storage/"+storageTypes[i])
v.SetDefault("storage."+storageTypes[i]+".perm", 0777)
// v.SetDefault("storage."+storageTypes[i]+".no_grow_sync", false)
// v.SetDefault("storage."+storageTypes[i]+".lock_timeout", "30s")
}
}
// Object section
{
v.SetDefault("object.max_processing_size", 100) // size in MB, use 0 to remove restriction
@ -274,20 +256,6 @@ func setDefaults(v *viper.Viper) {
)
}
{ // Reputation
// Put method name
v.SetDefault(
morph.ReputationContractPutOptPath(),
"Put",
)
// List method name
v.SetDefault(
morph.ReputationContractListOptPath(),
"List",
)
}
{ // Netmap
// AddPeer method name
v.SetDefault(

22
cmd/neofs-node/main.go
View file

@ -12,14 +12,14 @@ import (
"github.com/nspcc-dev/neofs-api-go/service"
state2 "github.com/nspcc-dev/neofs-api-go/state"
crypto "github.com/nspcc-dev/neofs-crypto"
"github.com/nspcc-dev/neofs-node/lib/fix"
"github.com/nspcc-dev/neofs-node/lib/fix/config"
"github.com/nspcc-dev/neofs-node/lib/fix/web"
"github.com/nspcc-dev/neofs-node/lib/fix/worker"
"github.com/nspcc-dev/neofs-node/lib/muxer"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/config"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/worker"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/node"
"github.com/nspcc-dev/neofs-node/misc"
"github.com/nspcc-dev/neofs-node/modules/node"
"github.com/nspcc-dev/neofs-node/services/public/state"
"github.com/nspcc-dev/neofs-node/pkg/network/muxer"
statesrv "github.com/nspcc-dev/neofs-node/pkg/network/transport/state/grpc"
"github.com/nspcc-dev/neofs-node/pkg/util/profiler"
"github.com/pkg/errors"
"github.com/spf13/viper"
"go.uber.org/dig"
@ -30,9 +30,9 @@ import (
type params struct {
dig.In
Debug web.Profiler `optional:"true"`
Metric web.Metrics `optional:"true"`
Worker worker.Workers `optional:"true"`
Debug profiler.Profiler `optional:"true"`
Metric profiler.Metrics `optional:"true"`
Worker worker.Workers `optional:"true"`
Muxer muxer.Mux
Logger *zap.Logger
}
@ -105,7 +105,7 @@ func runHealthCheck() {
grpc.WithInsecure())
check(err)
req := new(state.HealthRequest)
req := new(statesrv.HealthRequest)
req.SetTTL(service.NonForwardingTTL)
if err := service.SignRequestData(key, req); err != nil {
check(err)

26
modules/bootstrap/healthy.go → cmd/neofs-node/modules/bootstrap/healthy.go
View file

@ -2,12 +2,12 @@ package bootstrap
import (
"crypto/ecdsa"
"errors"
"sync"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/implementations"
"github.com/nspcc-dev/neofs-node/lib/placement"
"github.com/nspcc-dev/neofs-node/services/public/state"
contract "github.com/nspcc-dev/neofs-node/pkg/morph/client/netmap/wrapper"
state "github.com/nspcc-dev/neofs-node/pkg/network/transport/state/grpc"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
"github.com/spf13/viper"
"go.uber.org/dig"
"go.uber.org/zap"
@ -25,7 +25,7 @@ type (
// for ChangeState
PrivateKey *ecdsa.PrivateKey
MorphNetmapContract *implementations.MorphNetmapContract
Client *contract.Wrapper
}
healthyResult struct {
@ -47,9 +47,7 @@ type (
}
)
const (
errUnhealthy = internal.Error("unhealthy")
)
var errUnhealthy = errors.New("unhealthy")
func (h *healthyClient) setHandler(handler func() error) {
if handler == nil {
@ -71,12 +69,12 @@ func (h *healthyClient) Healthy() error {
func newHealthy(p healthyParams) (res healthyResult, err error) {
sp := state.Params{
Stater: p.Place,
Logger: p.Logger,
Viper: p.Viper,
Checkers: p.Checkers,
PrivateKey: p.PrivateKey,
MorphNetmapContract: p.MorphNetmapContract,
Stater: p.Place,
Logger: p.Logger,
Viper: p.Viper,
Checkers: p.Checkers,
PrivateKey: p.PrivateKey,
Client: p.Client,
}
if res.StateService, err = state.New(sp); err != nil {

4
modules/bootstrap/module.go → cmd/neofs-node/modules/bootstrap/module.go
View file

@ -1,8 +1,6 @@
package bootstrap
import (
"github.com/nspcc-dev/neofs-node/lib/fix/module"
)
import "github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/module"
// Module is a module of bootstrap component.
var Module = module.Module{

0
lib/fix/catch.go → cmd/neofs-node/modules/fix/catch.go
View file

0
lib/fix/config/config.go → cmd/neofs-node/modules/fix/config/config.go
View file

9
lib/fix/fix.go → cmd/neofs-node/modules/fix/fix.go
View file

@ -6,10 +6,11 @@ import (
"strconv"
"strings"
"github.com/nspcc-dev/neofs-node/lib/fix/config"
"github.com/nspcc-dev/neofs-node/lib/fix/logger"
"github.com/nspcc-dev/neofs-node/lib/fix/module"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/config"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/module"
"github.com/nspcc-dev/neofs-node/misc"
"github.com/nspcc-dev/neofs-node/pkg/util/grace"
"github.com/nspcc-dev/neofs-node/pkg/util/logger"
"github.com/pkg/errors"
"github.com/spf13/viper"
"go.uber.org/dig"
@ -90,7 +91,7 @@ func New(s *Settings, mod module.Module) App {
mod = mod.Append(
module.Module{
{Constructor: logger.NewLogger},
{Constructor: NewGracefulContext},
{Constructor: grace.NewGracefulContext},
{Constructor: func() (*viper.Viper, error) {
return config.NewConfig(config.Params{
File: s.File,

0
lib/fix/module/module.go → cmd/neofs-node/modules/fix/module/module.go
View file

0
lib/fix/services.go → cmd/neofs-node/modules/fix/services.go
View file

0
lib/fix/worker/worker.go → cmd/neofs-node/modules/fix/worker/worker.go
View file

0
modules/grpc/billing.go → cmd/neofs-node/modules/grpc/billing.go
View file

4
modules/grpc/module.go → cmd/neofs-node/modules/grpc/module.go
View file

@ -1,8 +1,6 @@
package grpc
import (
"github.com/nspcc-dev/neofs-node/lib/fix/module"
)
import "github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/module"
// Module is a gRPC layer module.
var Module = module.Module{

7
modules/grpc/routing.go → cmd/neofs-node/modules/grpc/routing.go
View file

@ -8,6 +8,7 @@ import (
middleware "github.com/grpc-ecosystem/go-grpc-middleware"
gZap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap"
prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
libgrpc "github.com/nspcc-dev/neofs-node/pkg/network/transport/grpc"
"github.com/spf13/viper"
"go.uber.org/dig"
"go.uber.org/zap"
@ -15,11 +16,7 @@ import (
)
type (
// Service interface
Service interface {
Name() string
Register(*grpc.Server)
}
Service = libgrpc.Service
// ServerParams to create gRPC-server
// and provide service-handlers

38
modules/morph/balance.go → cmd/neofs-node/modules/morph/balance.go
View file

@ -1,8 +1,9 @@
package morph
import (
"github.com/nspcc-dev/neofs-node/lib/implementations"
"github.com/nspcc-dev/neofs-node/services/public/accounting"
contract "github.com/nspcc-dev/neofs-node/pkg/morph/client/balance"
clientWrapper "github.com/nspcc-dev/neofs-node/pkg/morph/client/balance/wrapper"
accounting "github.com/nspcc-dev/neofs-node/pkg/network/transport/accounting/grpc"
"github.com/pkg/errors"
"go.uber.org/dig"
)
@ -10,7 +11,7 @@ import (
type balanceContractResult struct {
dig.Out
BalanceContract implementations.MorphBalanceContract
Client *clientWrapper.Wrapper
AccountingService accounting.Service
}
@ -41,27 +42,28 @@ func newBalanceContract(p contractParams) (res balanceContractResult, err error)
return
}
morphClient := implementations.MorphBalanceContract{}
morphClient.SetBalanceContractClient(client)
var (
balanceOfMethod = p.Viper.GetString(BalanceContractBalanceOfOptPath())
decimalsMethod = p.Viper.GetString(BalanceContractDecimalsOfOptPath())
)
morphClient.SetBalanceOfMethodName(
p.Viper.GetString(
BalanceContractBalanceOfOptPath(),
),
)
morphClient.SetDecimalsMethodName(
p.Viper.GetString(
BalanceContractDecimalsOfOptPath(),
),
)
var c *contract.Client
if c, err = contract.New(client,
contract.WithBalanceOfMethod(balanceOfMethod),
contract.WithDecimalsMethod(decimalsMethod),
); err != nil {
return
}
if res.Client, err = clientWrapper.New(c); err != nil {
return
}
if res.AccountingService, err = accounting.New(accounting.Params{
MorphBalanceContract: morphClient,
ContractClient: res.Client,
}); err != nil {
return
}
res.BalanceContract = morphClient
return
}

18
modules/morph/common.go → cmd/neofs-node/modules/morph/common.go
View file

@ -2,17 +2,16 @@ package morph
import (
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neofs-api-go/bootstrap"
"github.com/nspcc-dev/neofs-node/lib/blockchain/event"
"github.com/nspcc-dev/neofs-node/lib/blockchain/goclient"
"github.com/nspcc-dev/neofs-node/lib/implementations"
"github.com/nspcc-dev/neofs-node/pkg/core/netmap"
"github.com/nspcc-dev/neofs-node/pkg/morph/client"
"github.com/nspcc-dev/neofs-node/pkg/morph/event"
"github.com/spf13/viper"
"go.uber.org/dig"
"go.uber.org/zap"
)
// SmartContracts maps smart contract name to contract client.
type SmartContracts map[string]implementations.StaticContractClient
type SmartContracts map[string]*client.StaticClient
// EventHandlers maps notification event name to handler information.
type EventHandlers map[string]event.HandlerInfo
@ -22,7 +21,7 @@ type morphContractsParams struct {
Viper *viper.Viper
GoClient *goclient.Client
Client *client.Client
Listener event.Listener
}
@ -36,11 +35,11 @@ type contractParams struct {
MorphContracts SmartContracts
NodeInfo bootstrap.NodeInfo
NodeInfo netmap.Info
}
func newMorphContracts(p morphContractsParams) (SmartContracts, EventHandlers, error) {
mContracts := make(map[string]implementations.StaticContractClient, len(ContractNames))
mContracts := make(map[string]*client.StaticClient, len(ContractNames))
mHandlers := make(map[string]event.HandlerInfo)
for _, contractName := range ContractNames {
@ -59,7 +58,7 @@ func newMorphContracts(p morphContractsParams) (SmartContracts, EventHandlers, e
),
)
mContracts[contractName], err = implementations.NewStaticContractClient(p.GoClient, scHash, fee)
mContracts[contractName], err = client.NewStatic(p.Client, scHash, fee)
if err != nil {
return nil, nil, err
}
@ -109,7 +108,6 @@ const (
// ContractNames is a list of smart contract names.
var ContractNames = []string{
containerContractName,
reputationContractName,
NetmapContractName,
BalanceContractName,
}

77
modules/morph/container.go → cmd/neofs-node/modules/morph/container.go
View file

@ -1,9 +1,10 @@
package morph
import (
"github.com/nspcc-dev/neofs-node/lib/acl"
"github.com/nspcc-dev/neofs-node/lib/container"
"github.com/nspcc-dev/neofs-node/lib/implementations"
eacl "github.com/nspcc-dev/neofs-node/pkg/core/container/acl/extended/storage"
"github.com/nspcc-dev/neofs-node/pkg/core/container/storage"
contract "github.com/nspcc-dev/neofs-node/pkg/morph/client/container"
clientWrapper "github.com/nspcc-dev/neofs-node/pkg/morph/client/container/wrapper"
"github.com/pkg/errors"
"go.uber.org/dig"
)
@ -11,13 +12,9 @@ import (
type containerContractResult struct {
dig.Out
ContainerContract *implementations.MorphContainerContract
ExtendedACLStore eacl.Storage
BinaryExtendedACLStore acl.BinaryExtendedACLStore
ExtendedACLSource acl.ExtendedACLSource
ContainerStorage container.Storage
ContainerStorage storage.Storage
}
const (
@ -73,50 +70,34 @@ func newContainerContract(p contractParams) (res containerContractResult, err er
return
}
morphClient := new(implementations.MorphContainerContract)
morphClient.SetContainerContractClient(client)
morphClient.SetEACLSetMethodName(
p.Viper.GetString(
ContainerContractSetEACLOptPath(),
),
)
morphClient.SetEACLGetMethodName(
p.Viper.GetString(
ContainerContractEACLOptPath(),
),
)
morphClient.SetContainerGetMethodName(
p.Viper.GetString(
ContainerContractGetOptPath(),
),
)
morphClient.SetContainerPutMethodName(
p.Viper.GetString(
ContainerContractPutOptPath(),
),
)
morphClient.SetContainerDeleteMethodName(
p.Viper.GetString(
ContainerContractDelOptPath(),
),
)
morphClient.SetContainerListMethodName(
p.Viper.GetString(
ContainerContractListOptPath(),
),
var (
setEACLMethod = p.Viper.GetString(ContainerContractSetEACLOptPath())
eaclMethod = p.Viper.GetString(ContainerContractEACLOptPath())
getMethod = p.Viper.GetString(ContainerContractGetOptPath())
putMethod = p.Viper.GetString(ContainerContractPutOptPath())
deleteMethod = p.Viper.GetString(ContainerContractDelOptPath())
listMethod = p.Viper.GetString(ContainerContractListOptPath())
)
res.ContainerContract = morphClient
res.BinaryExtendedACLStore = morphClient
res.ExtendedACLSource, err = implementations.ExtendedACLSourceFromBinary(res.BinaryExtendedACLStore)
if err != nil {
var containerClient *contract.Client
if containerClient, err = contract.New(client,
contract.WithSetEACLMethod(setEACLMethod),
contract.WithEACLMethod(eaclMethod),
contract.WithGetMethod(getMethod),
contract.WithPutMethod(putMethod),
contract.WithDeleteMethod(deleteMethod),
contract.WithListMethod(listMethod),
); err != nil {
return
}
res.ContainerStorage = morphClient
var wrapClient *clientWrapper.Wrapper
if wrapClient, err = clientWrapper.New(containerClient); err != nil {
return
}
res.ContainerStorage = wrapClient
res.ExtendedACLStore = wrapClient
return res, nil
}

4
modules/morph/event.go → cmd/neofs-node/modules/morph/event.go
View file

@ -1,8 +1,8 @@
package morph
import (
"github.com/nspcc-dev/neofs-node/lib/blockchain/event"
"github.com/nspcc-dev/neofs-node/lib/blockchain/event/netmap"
"github.com/nspcc-dev/neofs-node/pkg/morph/event"
"github.com/nspcc-dev/neofs-node/pkg/morph/event/netmap"
)
const eventOpt = "event"

31
cmd/neofs-node/modules/morph/goclient.go Normal file
View file

@ -0,0 +1,31 @@
package morph
import (
"crypto/ecdsa"
"github.com/nspcc-dev/neo-go/pkg/config/netmode"
"github.com/nspcc-dev/neofs-node/pkg/morph/client"
"github.com/spf13/viper"
"go.uber.org/dig"
"go.uber.org/zap"
)
type morphClientParams struct {
dig.In
Viper *viper.Viper
Logger *zap.Logger
Key *ecdsa.PrivateKey
}
func newClient(p morphClientParams) (*client.Client, error) {
return client.New(
p.Key,
p.Viper.GetString(optPath(prefix, endpointOpt)),
client.WithLogger(p.Logger),
client.WithDialTimeout(p.Viper.GetDuration(optPath(prefix, dialTimeoutOpt))),
client.WithMagic(netmode.Magic(p.Viper.GetUint32(optPath(prefix, magicNumberOpt)))),
)
}

4
modules/morph/listener.go → cmd/neofs-node/modules/morph/listener.go
View file

@ -3,8 +3,8 @@ package morph
import (
"context"
"github.com/nspcc-dev/neofs-node/lib/blockchain/event"
"github.com/nspcc-dev/neofs-node/lib/blockchain/subscriber"
"github.com/nspcc-dev/neofs-node/pkg/morph/event"
"github.com/nspcc-dev/neofs-node/pkg/morph/subscriber"
"github.com/spf13/viper"
"go.uber.org/dig"
"go.uber.org/zap"

5
modules/morph/module.go → cmd/neofs-node/modules/morph/module.go
View file

@ -3,15 +3,14 @@ package morph
import (
"strings"
"github.com/nspcc-dev/neofs-node/lib/fix/module"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/module"
)
// Module is a Neo:Morph module.
var Module = module.Module{
{Constructor: newMorphClient},
{Constructor: newClient},
{Constructor: newMorphContracts},
{Constructor: newContainerContract},
{Constructor: newReputationContract},
{Constructor: newNetmapContract},
{Constructor: newEventListener},
{Constructor: newBalanceContract},

75
modules/morph/netmap.go → cmd/neofs-node/modules/morph/netmap.go
View file

@ -1,10 +1,9 @@
package morph
import (
"github.com/nspcc-dev/neofs-node/lib/boot"
"github.com/nspcc-dev/neofs-node/lib/implementations"
"github.com/nspcc-dev/neofs-node/lib/ir"
"github.com/nspcc-dev/neofs-node/lib/netmap"
contract "github.com/nspcc-dev/neofs-node/pkg/morph/client/netmap"
clientWrapper "github.com/nspcc-dev/neofs-node/pkg/morph/client/netmap/wrapper"
"github.com/nspcc-dev/neofs-node/pkg/network/bootstrap"
"github.com/pkg/errors"
"go.uber.org/dig"
)
@ -12,13 +11,9 @@ import (
type netmapContractResult struct {
dig.Out
NetmapContract *implementations.MorphNetmapContract
Client *clientWrapper.Wrapper
NetMapStorage netmap.Storage
IRStorage ir.Storage
StorageBootController boot.StorageBootController
NodeRegisterer *bootstrap.Registerer
}
const (
@ -68,48 +63,32 @@ func newNetmapContract(p contractParams) (res netmapContractResult, err error) {
return
}
morphClient := new(implementations.MorphNetmapContract)
morphClient.SetNetmapContractClient(client)
morphClient.SetAddPeerMethodName(
p.Viper.GetString(
NetmapContractAddPeerOptPath(),
),
)
morphClient.SetNewEpochMethodName(
p.Viper.GetString(
NetmapContractNewEpochOptPath(),
),
)
morphClient.SetNetMapMethodName(
p.Viper.GetString(
NetmapContractNetmapOptPath(),
),
)
morphClient.SetUpdateStateMethodName(
p.Viper.GetString(
NetmapContractUpdateStateOptPath(),
),
)
morphClient.SetIRListMethodName(
p.Viper.GetString(
NetmapContractIRListOptPath(),
),
var (
addPeerMethod = p.Viper.GetString(NetmapContractAddPeerOptPath())
newEpochMethod = p.Viper.GetString(NetmapContractNewEpochOptPath())
netmapMethod = p.Viper.GetString(NetmapContractNetmapOptPath())
updStateMethod = p.Viper.GetString(NetmapContractUpdateStateOptPath())
irListMethod = p.Viper.GetString(NetmapContractIRListOptPath())
)
bootCtrl := boot.StorageBootController{}
bootCtrl.SetPeerBootstrapper(morphClient)
bootCtrl.SetLogger(p.Logger)
var c *contract.Client
if c, err = contract.New(client,
contract.WithAddPeerMethod(addPeerMethod),
contract.WithNewEpochMethod(newEpochMethod),
contract.WithNetMapMethod(netmapMethod),
contract.WithUpdateStateMethod(updStateMethod),
contract.WithInnerRingListMethod(irListMethod),
); err != nil {
return
}
bootPrm := boot.StorageBootParams{}
bootPrm.SetNodeInfo(&p.NodeInfo)
if res.Client, err = clientWrapper.New(c); err != nil {
return
}
bootCtrl.SetBootParams(bootPrm)
res.StorageBootController = bootCtrl
res.NetmapContract = morphClient
res.NetMapStorage = morphClient
res.IRStorage = morphClient
if res.NodeRegisterer, err = bootstrap.New(res.Client, p.NodeInfo); err != nil {
return
}
return res, nil
}

2
modules/network/http.go → cmd/neofs-node/modules/network/http.go
View file

@ -2,7 +2,7 @@ package network
import (
"github.com/fasthttp/router"
svc "github.com/nspcc-dev/neofs-node/modules/bootstrap"
svc "github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/bootstrap"
"github.com/valyala/fasthttp"
"go.uber.org/dig"
)

9
modules/network/module.go → cmd/neofs-node/modules/network/module.go
View file

@ -1,8 +1,8 @@
package network
import (
"github.com/nspcc-dev/neofs-node/lib/fix/module"
"github.com/nspcc-dev/neofs-node/lib/fix/web"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/module"
"github.com/nspcc-dev/neofs-node/pkg/util/profiler"
)
// Module is a network layer module.
@ -10,11 +10,10 @@ var Module = module.Module{
{Constructor: newMuxer},
{Constructor: newPeers},
{Constructor: newPlacement},
{Constructor: newTransport},
// Metrics is prometheus handler
{Constructor: web.NewMetrics},
{Constructor: profiler.NewMetrics},
// Profiler is pprof handler
{Constructor: web.NewProfiler},
{Constructor: profiler.NewProfiler},
{Constructor: newHTTPHandler},
}

6
modules/network/muxer.go → cmd/neofs-node/modules/network/muxer.go
View file

@ -4,8 +4,7 @@ import (
"time"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-node/lib/muxer"
"github.com/nspcc-dev/neofs-node/lib/peers"
"github.com/nspcc-dev/neofs-node/pkg/network/muxer"
"github.com/spf13/viper"
"github.com/valyala/fasthttp"
"go.uber.org/dig"
@ -19,8 +18,6 @@ type muxerParams struct {
Logger *zap.Logger
P2P *grpc.Server
Peers peers.Interface
Address multiaddr.Multiaddr
ShutdownTTL time.Duration `name:"shutdown_ttl"`
API fasthttp.RequestHandler
@ -48,7 +45,6 @@ func newFastHTTPServer(p muxerParams) *fasthttp.Server {
func newMuxer(p muxerParams) muxer.Mux {
return muxer.New(muxer.Params{
P2P: p.P2P,
Peers: p.Peers,
Logger: p.Logger,
Address: p.Address,
ShutdownTTL: p.ShutdownTTL,

21
modules/network/peers.go → cmd/neofs-node/modules/network/peers.go
View file

@ -2,8 +2,7 @@ package network
import (
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-node/lib/peers"
"github.com/nspcc-dev/neofs-node/lib/transport"
"github.com/nspcc-dev/neofs-node/pkg/network/peers"
"github.com/spf13/viper"
"go.uber.org/dig"
"go.uber.org/zap"
@ -12,26 +11,14 @@ import (
type peersParams struct {
dig.In
Viper *viper.Viper
Logger *zap.Logger
Address multiaddr.Multiaddr
Transport transport.Transport
}
func newTransport(v *viper.Viper) transport.Transport {
return transport.New(
v.GetInt64("transport.attempts_count"),
v.GetDuration("transport.attempts_ttl"),
)
Viper *viper.Viper
Logger *zap.Logger
Address multiaddr.Multiaddr
}
func newPeers(p peersParams) (peers.Interface, error) {
return peers.New(peers.Params{
Logger: p.Logger,
Address: p.Address,
Transport: p.Transport,
Attempts: p.Viper.GetInt64("peers.attempts_count"),
AttemptsTTL: p.Viper.GetDuration("peers.attempts_ttl"),
ConnectionTTL: p.Viper.GetDuration("peers.connections_ttl"),
ConnectionIDLE: p.Viper.GetDuration("peers.connections_idle"),
MetricsTimeout: p.Viper.GetDuration("peers.metrics_timeout"),

24
modules/network/placement.go → cmd/neofs-node/modules/network/placement.go
View file

@ -1,14 +1,14 @@
package network
import (
"github.com/nspcc-dev/neofs-node/lib/blockchain/event"
netmapevent "github.com/nspcc-dev/neofs-node/lib/blockchain/event/netmap"
libcnr "github.com/nspcc-dev/neofs-node/lib/container"
"github.com/nspcc-dev/neofs-node/lib/netmap"
"github.com/nspcc-dev/neofs-node/lib/peers"
"github.com/nspcc-dev/neofs-node/lib/placement"
"github.com/nspcc-dev/neofs-node/modules/morph"
"github.com/nspcc-dev/neofs-node/services/public/state"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/morph"
"github.com/nspcc-dev/neofs-node/pkg/core/container/storage"
contract "github.com/nspcc-dev/neofs-node/pkg/morph/client/netmap/wrapper"
"github.com/nspcc-dev/neofs-node/pkg/morph/event"
netmapevent "github.com/nspcc-dev/neofs-node/pkg/morph/event/netmap"
"github.com/nspcc-dev/neofs-node/pkg/network/peers"
state "github.com/nspcc-dev/neofs-node/pkg/network/transport/state/grpc"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
"go.uber.org/dig"
"go.uber.org/zap"
)
@ -19,11 +19,11 @@ type (
Log *zap.Logger
Peers peers.Store
Fetcher libcnr.Storage
Fetcher storage.Storage
MorphEventListener event.Listener
NetMapStorage netmap.Storage
NetMapClient *contract.Wrapper
MorphEventHandlers morph.EventHandlers
}
@ -51,7 +51,7 @@ func newPlacement(p placementParams) placementOutput {
morph.NewEpochEventType,
)]; ok {
handlerInfo.SetHandler(func(ev event.Event) {
nmRes, err := p.NetMapStorage.GetNetMap(netmap.GetParams{})
nm, err := p.NetMapClient.GetNetMap()
if err != nil {
p.Log.Error("could not get network map",
zap.String("error", err.Error()),
@ -61,7 +61,7 @@ func newPlacement(p placementParams) placementOutput {
if err := place.Update(
ev.(netmapevent.NewEpoch).EpochNumber(),
nmRes.NetMap(),
nm,
); err != nil {
p.Log.Error("could not update network map in placement component",
zap.String("error", err.Error()),

18
modules/node/audit.go → cmd/neofs-node/modules/node/audit.go
View file

@ -4,9 +4,11 @@ import (
"crypto/ecdsa"
"github.com/nspcc-dev/neofs-api-go/session"
"github.com/nspcc-dev/neofs-node/lib/implementations"
"github.com/nspcc-dev/neofs-node/lib/peers"
"github.com/nspcc-dev/neofs-node/services/public/object"
"github.com/nspcc-dev/neofs-node/pkg/network/peers"
object "github.com/nspcc-dev/neofs-node/pkg/network/transport/object/grpc"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/replication/storage"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/transport"
"github.com/spf13/viper"
"go.uber.org/zap"
)
@ -15,7 +17,7 @@ type (
cnrHandlerParams struct {
*viper.Viper
*zap.Logger
Placer implementations.ObjectPlacer
Placer *placement.PlacementWrapper
PeerStore peers.Store
Peers peers.Interface
TimeoutsPrefix string
@ -25,8 +27,8 @@ type (
}
)
func newObjectsContainerHandler(p cnrHandlerParams) (implementations.SelectiveContainerExecutor, error) {
as, err := implementations.NewAddressStore(p.PeerStore, p.Logger)
func newObjectsContainerHandler(p cnrHandlerParams) (transport.SelectiveContainerExecutor, error) {
as, err := storage.NewAddressStore(p.PeerStore, p.Logger)
if err != nil {
return nil, err
}
@ -50,12 +52,12 @@ func newObjectsContainerHandler(p cnrHandlerParams) (implementations.SelectiveCo
return nil, err
}
exec, err := implementations.NewContainerTraverseExecutor(multiTransport)
exec, err := transport.NewContainerTraverseExecutor(multiTransport)
if err != nil {
return nil, err
}
return implementations.NewObjectContainerHandler(implementations.ObjectContainerHandlerParams{
return transport.NewObjectContainerHandler(transport.ObjectContainerHandlerParams{
NodeLister: p.Placer,
Executor: exec,
Logger: p.Logger,

12
modules/node/container.go → cmd/neofs-node/modules/node/container.go
View file

@ -1,10 +1,10 @@
package node
import (
"github.com/nspcc-dev/neofs-node/lib/acl"
libcnr "github.com/nspcc-dev/neofs-node/lib/container"
svc "github.com/nspcc-dev/neofs-node/modules/bootstrap"
"github.com/nspcc-dev/neofs-node/services/public/container"
svc "github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/bootstrap"
eacl "github.com/nspcc-dev/neofs-node/pkg/core/container/acl/extended/storage"
"github.com/nspcc-dev/neofs-node/pkg/core/container/storage"
container "github.com/nspcc-dev/neofs-node/pkg/network/transport/container/grpc"
"go.uber.org/dig"
"go.uber.org/zap"
)
@ -16,9 +16,9 @@ type cnrParams struct {
Healthy svc.HealthyClient
ExtendedACLStore acl.BinaryExtendedACLStore
ExtendedACLStore eacl.Storage
ContainerStorage libcnr.Storage
ContainerStorage storage.Storage
}
func newContainerService(p cnrParams) (container.Service, error) {

35
cmd/neofs-node/modules/node/core.go Normal file
View file

@ -0,0 +1,35 @@
package node
import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/bucket"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/bucket/boltdb"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/bucket/fsbucket"
"github.com/spf13/viper"
)
type Buckets map[string]bucket.Bucket
const (
fsBucket = "fsbucket"
boltBucket = "bolt"
)
func newBuckets(v *viper.Viper) (Buckets, error) {
var (
err error
mBuckets = make(Buckets)
)
if mBuckets[fsBucket], err = fsbucket.NewBucket(v); err != nil {
return nil, err
}
boltOpts, err := boltdb.NewOptions(v)
if err != nil {
return nil, err
} else if mBuckets[boltBucket], err = boltdb.NewBucket(&boltOpts); err != nil {
return nil, err
}
return mBuckets, nil
}

29
modules/node/localstore.go → cmd/neofs-node/modules/node/localstore.go
View file

@ -1,10 +1,9 @@
package node
import (
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/nspcc-dev/neofs-node/lib/localstore"
"github.com/nspcc-dev/neofs-node/lib/meta"
"github.com/nspcc-dev/neofs-node/lib/metrics"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/localstore"
meta2 "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/meta"
metrics2 "github.com/nspcc-dev/neofs-node/pkg/services/metrics"
"go.uber.org/atomic"
"go.uber.org/dig"
"go.uber.org/zap"
@ -15,9 +14,9 @@ type (
dig.In
Logger *zap.Logger
Storage core.Storage
Buckets Buckets
Counter *atomic.Float64
Collector metrics.Collector
Collector metrics2.Collector
}
metaIterator struct {
@ -25,30 +24,20 @@ type (
}
)
func newMetaIterator(iter localstore.Iterator) meta.Iterator {
func newMetaIterator(iter localstore.Iterator) meta2.Iterator {
return &metaIterator{iter: iter}
}
func (m *metaIterator) Iterate(handler meta.IterateFunc) error {
func (m *metaIterator) Iterate(handler meta2.IterateFunc) error {
return m.iter.Iterate(nil, func(objMeta *localstore.ObjectMeta) bool {
return handler == nil || handler(objMeta.Object) != nil
})
}
func newLocalstore(p localstoreParams) (localstore.Localstore, error) {
metaBucket, err := p.Storage.GetBucket(core.MetaStore)
if err != nil {
return nil, err
}
blobBucket, err := p.Storage.GetBucket(core.BlobStore)
if err != nil {
return nil, err
}
local, err := localstore.New(localstore.Params{
BlobBucket: blobBucket,
MetaBucket: metaBucket,
BlobBucket: p.Buckets[fsBucket],
MetaBucket: p.Buckets[boltBucket],
Logger: p.Logger,
Collector: p.Collector,
})

24
modules/node/metrics.go → cmd/neofs-node/modules/node/metrics.go
View file

@ -1,9 +1,8 @@
package node
import (
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/nspcc-dev/neofs-node/lib/metrics"
mService "github.com/nspcc-dev/neofs-node/services/metrics"
metrics "github.com/nspcc-dev/neofs-node/pkg/network/transport/metrics/grpc"
metrics2 "github.com/nspcc-dev/neofs-node/pkg/services/metrics"
"github.com/spf13/viper"
"go.uber.org/atomic"
"go.uber.org/dig"
@ -17,36 +16,31 @@ type (
Logger *zap.Logger
Options []string `name:"node_options"`
Viper *viper.Viper
Store core.Storage
Buckets Buckets
}
metricsServiceParams struct {
dig.In
Logger *zap.Logger
Collector metrics.Collector
Collector metrics2.Collector
}
)
func newObjectCounter() *atomic.Float64 { return atomic.NewFloat64(0) }
func newMetricsService(p metricsServiceParams) (mService.Service, error) {
return mService.New(mService.Params{
func newMetricsService(p metricsServiceParams) (metrics.Service, error) {
return metrics.New(metrics.Params{
Logger: p.Logger,
Collector: p.Collector,
})
}
func newMetricsCollector(p metricsParams) (metrics.Collector, error) {
store, err := p.Store.GetBucket(core.SpaceMetricsStore)
if err != nil {
return nil, err
}
return metrics.New(metrics.Params{
func newMetricsCollector(p metricsParams) (metrics2.Collector, error) {
return metrics2.New(metrics2.Params{
Options: p.Options,
Logger: p.Logger,
Interval: p.Viper.GetDuration("metrics_collector.interval"),
MetricsStore: store,
MetricsStore: p.Buckets[fsBucket],
})
}

36
modules/node/module.go → cmd/neofs-node/modules/node/module.go
View file

@ -2,20 +2,19 @@ package node
import (
"github.com/nspcc-dev/neofs-api-go/session"
"github.com/nspcc-dev/neofs-node/lib/blockchain/event"
"github.com/nspcc-dev/neofs-node/lib/boot"
"github.com/nspcc-dev/neofs-node/lib/fix/module"
"github.com/nspcc-dev/neofs-node/lib/fix/worker"
"github.com/nspcc-dev/neofs-node/lib/metrics"
"github.com/nspcc-dev/neofs-node/lib/netmap"
"github.com/nspcc-dev/neofs-node/lib/peers"
"github.com/nspcc-dev/neofs-node/lib/replication"
"github.com/nspcc-dev/neofs-node/modules/bootstrap"
"github.com/nspcc-dev/neofs-node/modules/grpc"
"github.com/nspcc-dev/neofs-node/modules/morph"
"github.com/nspcc-dev/neofs-node/modules/network"
"github.com/nspcc-dev/neofs-node/modules/settings"
"github.com/nspcc-dev/neofs-node/modules/workers"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/bootstrap"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/module"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/worker"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/grpc"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/morph"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/network"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/settings"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/workers"
"github.com/nspcc-dev/neofs-node/pkg/morph/event"
libboot "github.com/nspcc-dev/neofs-node/pkg/network/bootstrap"
"github.com/nspcc-dev/neofs-node/pkg/network/peers"
metrics2 "github.com/nspcc-dev/neofs-node/pkg/services/metrics"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/replication"
"github.com/spf13/viper"
"go.uber.org/dig"
"go.uber.org/zap"
@ -30,11 +29,11 @@ type jobParams struct {
Replicator replication.Manager
PeersInterface peers.Interface
Metrics metrics.Collector
Metrics metrics2.Collector
MorphEventListener event.Listener
StorageBootController boot.StorageBootController
NodeRegisterer *libboot.Registerer
}
// Module is a NeoFS node module.
@ -42,8 +41,7 @@ var Module = module.Module{
{Constructor: attachJobs},
{Constructor: newPeerstore},
{Constructor: attachServices},
{Constructor: netmap.NewNetmap},
{Constructor: newStorage},
{Constructor: newBuckets},
{Constructor: newMetricsCollector},
{Constructor: newObjectCounter},
@ -86,6 +84,6 @@ func attachJobs(p jobParams) worker.Jobs {
"metrics": p.Metrics.Start,
"event_listener": p.MorphEventListener.Listen,
"replicator": p.Replicator.Process,
"boot": p.StorageBootController.Bootstrap,
"boot": p.NodeRegisterer.Bootstrap,
}
}

88
modules/node/objectmanager.go → cmd/neofs-node/modules/node/objectmanager.go
View file

@ -7,16 +7,17 @@ import (
"github.com/nspcc-dev/neofs-api-go/hash"
apiobj "github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/session"
libacl "github.com/nspcc-dev/neofs-node/lib/acl"
"github.com/nspcc-dev/neofs-node/lib/container"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/nspcc-dev/neofs-node/lib/implementations"
"github.com/nspcc-dev/neofs-node/lib/ir"
"github.com/nspcc-dev/neofs-node/lib/localstore"
"github.com/nspcc-dev/neofs-node/lib/peers"
"github.com/nspcc-dev/neofs-node/lib/placement"
"github.com/nspcc-dev/neofs-node/lib/transformer"
"github.com/nspcc-dev/neofs-node/services/public/object"
eacl "github.com/nspcc-dev/neofs-node/pkg/core/container/acl/extended/storage"
"github.com/nspcc-dev/neofs-node/pkg/core/container/storage"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/localstore"
contract "github.com/nspcc-dev/neofs-node/pkg/morph/client/netmap/wrapper"
"github.com/nspcc-dev/neofs-node/pkg/network/peers"
object "github.com/nspcc-dev/neofs-node/pkg/network/transport/object/grpc"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
storage2 "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/replication/storage"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/transformer"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/transport"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/transport/storagegroup"
"github.com/spf13/viper"
"go.uber.org/dig"
"go.uber.org/zap"
@ -38,22 +39,18 @@ type (
Options []string `name:"node_options"`
Key *ecdsa.PrivateKey
IRStorage ir.Storage
NetMapClient *contract.Wrapper
EpochReceiver implementations.EpochReceiver
Placer *placement.PlacementWrapper
Placer implementations.ObjectPlacer
ExtendedACLStore eacl.Storage
ExtendedACLStore libacl.ExtendedACLSource
ContainerStorage container.Storage
ContainerStorage storage.Storage
}
)
const (
transformersSectionPath = "object.transformers."
aclMandatorySetBits = 0x04040444
)
const xorSalitor = "xor"
@ -65,7 +62,7 @@ func newObjectManager(p objectManagerParams) (object.Service, error) {
sltr = hash.SaltXOR
}
as, err := implementations.NewAddressStore(p.Peers, p.Logger)
as, err := storage2.NewAddressStore(p.Peers, p.Logger)
if err != nil {
return nil, err
}
@ -81,7 +78,7 @@ func newObjectManager(p objectManagerParams) (object.Service, error) {
tr, err := object.NewMultiTransport(object.MultiTransportParams{
AddressStore: as,
EpochReceiver: p.EpochReceiver,
EpochReceiver: p.Placer,
RemoteService: rs,
Logger: p.Logger,
Key: p.Key,
@ -98,12 +95,12 @@ func newObjectManager(p objectManagerParams) (object.Service, error) {
return nil, err
}
exec, err := implementations.NewContainerTraverseExecutor(tr)
exec, err := transport.NewContainerTraverseExecutor(tr)
if err != nil {
return nil, err
}
selectiveExec, err := implementations.NewObjectContainerHandler(implementations.ObjectContainerHandlerParams{
selectiveExec, err := transport.NewObjectContainerHandler(transport.ObjectContainerHandlerParams{
NodeLister: p.Placer,
Executor: exec,
Logger: p.Logger,
@ -112,7 +109,7 @@ func newObjectManager(p objectManagerParams) (object.Service, error) {
return nil, err
}
sgInfoRecv, err := implementations.NewStorageGroupInfoReceiver(implementations.StorageGroupInfoReceiverParams{
sgInfoRecv, err := storagegroup.NewStorageGroupInfoReceiver(storagegroup.StorageGroupInfoReceiverParams{
SelectiveContainerExecutor: selectiveExec,
Logger: p.Logger,
})
@ -120,16 +117,14 @@ func newObjectManager(p objectManagerParams) (object.Service, error) {
return nil, err
}
verifier, err := implementations.NewLocalIntegrityVerifier(
core.NewNeoKeyVerifier(),
)
verifier, err := storage2.NewLocalIntegrityVerifier()
if err != nil {
return nil, err
}
trans, err := transformer.NewTransformer(transformer.Params{
SGInfoReceiver: sgInfoRecv,
EpochReceiver: p.EpochReceiver,
EpochReceiver: p.Placer,
SizeLimit: uint64(p.Viper.GetInt64(transformersSectionPath+"payload_limiter.max_payload_size") * apiobj.UnitsKB),
Verifier: verifier,
})
@ -137,16 +132,7 @@ func newObjectManager(p objectManagerParams) (object.Service, error) {
return nil, err
}
aclChecker := libacl.NewMaskedBasicACLChecker(aclMandatorySetBits, libacl.DefaultAndFilter)
aclHelper, err := implementations.NewACLHelper(p.ContainerStorage)
if err != nil {
return nil, err
}
verifier, err = implementations.NewLocalHeadIntegrityVerifier(
core.NewNeoKeyVerifier(),
)
verifier, err = storage2.NewLocalHeadIntegrityVerifier()
if err != nil {
return nil, err
}
@ -163,16 +149,16 @@ func newObjectManager(p objectManagerParams) (object.Service, error) {
ObjectRestorer: transformer.NewRestorePipeline(
transformer.SplitRestorer(),
),
RemoteService: rs,
AddressStore: as,
Logger: p.Logger,
TokenStore: p.TokenStore,
EpochReceiver: p.EpochReceiver,
ContainerNodesLister: p.Placer,
Key: p.Key,
CheckACL: p.Viper.GetBool("object.check_acl"),
DialTimeout: p.Viper.GetDuration("object.dial_timeout"),
MaxPayloadSize: p.Viper.GetUint64("object.transformers.payload_limiter.max_payload_size") * uint64(apiobj.UnitsKB),
RemoteService: rs,
AddressStore: as,
Logger: p.Logger,
TokenStore: p.TokenStore,
EpochReceiver: p.Placer,
PlacementWrapper: p.Placer,
Key: p.Key,
CheckACL: p.Viper.GetBool("object.check_acl"),
DialTimeout: p.Viper.GetDuration("object.dial_timeout"),
MaxPayloadSize: p.Viper.GetUint64("object.transformers.payload_limiter.max_payload_size") * uint64(apiobj.UnitsKB),
PutParams: object.OperationParams{
Timeout: pto,
LogErrors: p.Viper.GetBool("object.put.log_errs"),
@ -205,15 +191,11 @@ func newObjectManager(p objectManagerParams) (object.Service, error) {
WindowSize: p.Viper.GetInt("object.window_size"),
ACLHelper: aclHelper,
BasicACLChecker: aclChecker,
IRStorage: p.IRStorage,
ContainerLister: p.Placer,
ContainerStorage: p.ContainerStorage,
NetmapClient: p.NetMapClient,
SGInfoReceiver: sgInfoRecv,
OwnerKeyVerifier: core.NewNeoKeyVerifier(),
ExtendedACLSource: p.ExtendedACLStore,
})
}

2
modules/node/peerstore.go → cmd/neofs-node/modules/node/peerstore.go
View file

@ -4,7 +4,7 @@ import (
"crypto/ecdsa"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-node/lib/peers"
"github.com/nspcc-dev/neofs-node/pkg/network/peers"
"go.uber.org/dig"
"go.uber.org/zap"
)

28
cmd/neofs-node/modules/node/placement.go Normal file
View file

@ -0,0 +1,28 @@
package node
import (
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
"go.uber.org/dig"
)
type (
placementToolParams struct {
dig.In
Placement placement.Component
}
placementToolResult struct {
dig.Out
Placer *placement.PlacementWrapper
}
)
func newPlacementTool(p placementToolParams) (res placementToolResult, err error) {
if res.Placer, err = placement.NewObjectPlacer(p.Placement); err != nil {
return
}
return
}

51
modules/node/replication.go → cmd/neofs-node/modules/node/replication.go
View file

@ -6,16 +6,14 @@ import (
"github.com/nspcc-dev/neofs-api-go/hash"
"github.com/nspcc-dev/neofs-api-go/session"
"github.com/nspcc-dev/neofs-node/lib/blockchain/event"
"github.com/nspcc-dev/neofs-node/lib/blockchain/event/netmap"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/nspcc-dev/neofs-node/lib/implementations"
"github.com/nspcc-dev/neofs-node/lib/ir"
"github.com/nspcc-dev/neofs-node/lib/localstore"
"github.com/nspcc-dev/neofs-node/lib/peers"
"github.com/nspcc-dev/neofs-node/lib/placement"
"github.com/nspcc-dev/neofs-node/lib/replication"
"github.com/nspcc-dev/neofs-node/modules/morph"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/morph"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/localstore"
"github.com/nspcc-dev/neofs-node/pkg/morph/event"
"github.com/nspcc-dev/neofs-node/pkg/morph/event/netmap"
"github.com/nspcc-dev/neofs-node/pkg/network/peers"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/replication"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/replication/storage"
"github.com/pkg/errors"
"github.com/spf13/viper"
"go.uber.org/dig"
@ -34,10 +32,9 @@ type (
Peers peers.Store
Placement placement.Component
Logger *zap.Logger
Lister ir.Storage
Key *ecdsa.PrivateKey
Placer implementations.ObjectPlacer
Placer *placement.PlacementWrapper
TokenStore session.PrivateTokenStore
@ -57,7 +54,7 @@ const (
)
func newReplicationManager(p replicationManagerParams) (replication.Manager, error) {
as, err := implementations.NewAddressStore(p.Peers, p.Logger)
as, err := storage.NewAddressStore(p.Peers, p.Logger)
if err != nil {
return nil, err
}
@ -80,14 +77,12 @@ func newReplicationManager(p replicationManagerParams) (replication.Manager, err
return nil, err
}
integrityVerifier, err := implementations.NewLocalIntegrityVerifier(
core.NewNeoKeyVerifier(),
)
integrityVerifier, err := storage.NewLocalIntegrityVerifier()
if err != nil {
return nil, err
}
verifier, err := implementations.NewObjectValidator(&implementations.ObjectValidatorParams{
verifier, err := storage.NewObjectValidator(&storage.ObjectValidatorParams{
AddressStore: ms,
Localstore: p.LocalStore,
Logger: p.Logger,
@ -183,7 +178,7 @@ func newPlacementHonorer(p replicationManagerParams, rss replication.RemoteStora
return nil, err
}
storage, err := implementations.NewObjectStorage(implementations.ObjectStorageParams{
storage, err := storage.NewObjectStorage(storage.ObjectStorageParams{
Localstore: p.LocalStore,
SelectiveContainerExecutor: och,
Logger: p.Logger,
@ -221,7 +216,7 @@ func newLocationDetector(p replicationManagerParams, ms replication.MultiSolver)
return nil, err
}
locator, err := implementations.NewObjectLocator(implementations.LocatorParams{
locator, err := storage.NewObjectLocator(storage.LocatorParams{
SelectiveContainerExecutor: och,
Logger: p.Logger,
})
@ -243,7 +238,7 @@ func newLocationDetector(p replicationManagerParams, ms replication.MultiSolver)
func newStorageValidator(p replicationManagerParams, as replication.AddressStore) (replication.StorageValidator, error) {
prefix := mainReplicationPrefix + "." + storageValidatorPrefix
var sltr implementations.Salitor
var sltr storage.Salitor
switch v := p.Viper.GetString(prefix + ".salitor"); v {
case xorSalitor:
@ -267,14 +262,12 @@ func newStorageValidator(p replicationManagerParams, as replication.AddressStore
return nil, err
}
headVerifier, err := implementations.NewLocalHeadIntegrityVerifier(
core.NewNeoKeyVerifier(),
)
headVerifier, err := storage.NewLocalHeadIntegrityVerifier()
if err != nil {
return nil, err
}
verifier, err := implementations.NewObjectValidator(&implementations.ObjectValidatorParams{
verifier, err := storage.NewObjectValidator(&storage.ObjectValidatorParams{
AddressStore: as,
Localstore: p.LocalStore,
SelectiveContainerExecutor: och,
@ -317,7 +310,7 @@ func newObjectReplicator(p replicationManagerParams, rss replication.RemoteStora
return nil, err
}
storage, err := implementations.NewObjectStorage(implementations.ObjectStorageParams{
storage, err := storage.NewObjectStorage(storage.ObjectStorageParams{
Localstore: p.LocalStore,
SelectiveContainerExecutor: och,
Logger: p.Logger,
@ -355,14 +348,12 @@ func newRestorer(p replicationManagerParams, ms replication.MultiSolver) (replic
return nil, err
}
integrityVerifier, err := implementations.NewLocalIntegrityVerifier(
core.NewNeoKeyVerifier(),
)
integrityVerifier, err := storage.NewLocalIntegrityVerifier()
if err != nil {
return nil, err
}
verifier, err := implementations.NewObjectValidator(&implementations.ObjectValidatorParams{
verifier, err := storage.NewObjectValidator(&storage.ObjectValidatorParams{
AddressStore: ms,
Localstore: p.LocalStore,
SelectiveContainerExecutor: och,
@ -373,7 +364,7 @@ func newRestorer(p replicationManagerParams, ms replication.MultiSolver) (replic
return nil, err
}
storage, err := implementations.NewObjectStorage(implementations.ObjectStorageParams{
storage, err := storage.NewObjectStorage(storage.ObjectStorageParams{
Localstore: p.LocalStore,
Logger: p.Logger,
})

36
cmd/neofs-node/modules/node/services.go Normal file
View file

@ -0,0 +1,36 @@
package node
import (
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/grpc"
accounting "github.com/nspcc-dev/neofs-node/pkg/network/transport/accounting/grpc"
container "github.com/nspcc-dev/neofs-node/pkg/network/transport/container/grpc"
metrics "github.com/nspcc-dev/neofs-node/pkg/network/transport/metrics/grpc"
object "github.com/nspcc-dev/neofs-node/pkg/network/transport/object/grpc"
session "github.com/nspcc-dev/neofs-node/pkg/network/transport/session/grpc"
state "github.com/nspcc-dev/neofs-node/pkg/network/transport/state/grpc"
"go.uber.org/dig"
)
type servicesParams struct {
dig.In
Status state.Service
Container container.Service
Object object.Service
Session session.Service
Accounting accounting.Service
Metrics metrics.Service
}
func attachServices(p servicesParams) grpc.ServicesResult {
return grpc.ServicesResult{
Services: []grpc.Service{
p.Status,
p.Container,
p.Accounting,
p.Metrics,
p.Session,
p.Object,
},
}
}

6
modules/node/session.go → cmd/neofs-node/modules/node/session.go
View file

@ -1,8 +1,8 @@
package node
import (
"github.com/nspcc-dev/neofs-node/lib/implementations"
"github.com/nspcc-dev/neofs-node/services/public/session"
session "github.com/nspcc-dev/neofs-node/pkg/network/transport/session/grpc"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
"go.uber.org/dig"
"go.uber.org/zap"
)
@ -14,7 +14,7 @@ type sessionParams struct {
TokenStore session.TokenStore
EpochReceiver implementations.EpochReceiver
EpochReceiver *placement.PlacementWrapper
}
func newSessionService(p sessionParams) (session.Service, error) {

9
modules/settings/address.go → cmd/neofs-node/modules/settings/address.go
View file

@ -6,7 +6,6 @@ import (
"strings"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/pkg/errors"
)
@ -21,10 +20,10 @@ const emptyAddr = "0.0.0.0"
const ip4ColonCount = 1
var (
errEmptyAddress = internal.Error("`node.address` could not be empty")
errEmptyProtocol = internal.Error("`node.protocol` could not be empty")
errUnknownProtocol = internal.Error("`node.protocol` unknown protocol")
errEmptyShutdownTTL = internal.Error("`node.shutdown_ttl` could not be empty")
errEmptyAddress = errors.New("`node.address` could not be empty")
errEmptyProtocol = errors.New("`node.protocol` could not be empty")
errUnknownProtocol = errors.New("`node.protocol` unknown protocol")
errEmptyShutdownTTL = errors.New("`node.shutdown_ttl` could not be empty")
)
func ipVersion(address string) string {

4
modules/settings/module.go → cmd/neofs-node/modules/settings/module.go
View file

@ -1,8 +1,6 @@
package settings
import (
"github.com/nspcc-dev/neofs-node/lib/fix/module"
)
import "github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/module"
// Module is a node settings module.
var Module = module.Module{

17
modules/settings/node.go → cmd/neofs-node/modules/settings/node.go
View file

@ -10,9 +10,9 @@ import (
"time"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-api-go/bootstrap"
crypto "github.com/nspcc-dev/neofs-crypto"
"github.com/nspcc-dev/neofs-node/lib/peers"
"github.com/nspcc-dev/neofs-node/pkg/core/netmap"
"github.com/nspcc-dev/neofs-node/pkg/network/peers"
"github.com/pkg/errors"
"github.com/spf13/viper"
"go.uber.org/dig"
@ -28,7 +28,7 @@ type (
NodeOpts []string `name:"node_options"`
ShutdownTTL time.Duration `name:"shutdown_ttl"`
NodeInfo bootstrap.NodeInfo
NodeInfo netmap.Info
}
)
@ -136,11 +136,12 @@ loop:
cfg.NodeOpts = append(cfg.NodeOpts, val)
}
cfg.NodeInfo = bootstrap.NodeInfo{
Address: cfg.Address.String(),
PubKey: crypto.MarshalPublicKey(&cfg.PrivateKey.PublicKey),
Options: cfg.NodeOpts,
}
nodeInfo := netmap.Info{}
nodeInfo.SetAddress(cfg.Address.String())
nodeInfo.SetPublicKey(crypto.MarshalPublicKey(&cfg.PrivateKey.PublicKey))
nodeInfo.SetOptions(cfg.NodeOpts)
cfg.NodeInfo = nodeInfo
l.Debug("loaded node options",
zap.Strings("options", cfg.NodeOpts))

4
modules/workers/module.go → cmd/neofs-node/modules/workers/module.go
View file

@ -1,8 +1,6 @@
package workers
import (
"github.com/nspcc-dev/neofs-node/lib/fix/module"
)
import "github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/module"
// Module is a workers module.
var Module = module.Module{

2
modules/workers/prepare.go → cmd/neofs-node/modules/workers/prepare.go
View file

@ -4,7 +4,7 @@ import (
"context"
"time"
"github.com/nspcc-dev/neofs-node/lib/fix/worker"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/modules/fix/worker"
"github.com/spf13/viper"
"go.uber.org/dig"
"go.uber.org/zap"

8
go.mod
View file

@ -4,8 +4,6 @@ go 1.14
require (
bou.ke/monkey v1.0.2
github.com/cenk/backoff v2.2.1+incompatible // indirect
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
github.com/fasthttp/router v1.0.2
github.com/gogo/protobuf v1.3.1
github.com/golang/protobuf v1.4.2
@ -17,15 +15,13 @@ require (
github.com/multiformats/go-multiaddr-net v0.1.2 // v0.1.1 => v0.1.2
github.com/multiformats/go-multihash v0.0.13
github.com/nspcc-dev/hrw v1.0.9
github.com/nspcc-dev/neo-go v0.90.0-pre.0.20200708064050-cf1e5243b90b
github.com/nspcc-dev/neofs-api-go v1.2.0
github.com/nspcc-dev/neo-go v0.90.0
github.com/nspcc-dev/neofs-api-go v1.3.0
github.com/nspcc-dev/neofs-crypto v0.3.0
github.com/nspcc-dev/netmap v1.7.0
github.com/panjf2000/ants/v2 v2.3.0
github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea // indirect
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.6.0
github.com/rubyist/circuitbreaker v2.2.1+incompatible
github.com/soheilhy/cmux v0.1.4
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/pflag v1.0.5 // indirect

BIN
go.sum
View file

Binary file not shown.

7
internal/error.go
View file

@ -1,7 +0,0 @@
package internal
// Error is a custom error.
type Error string
// Error is an implementation of error interface.
func (e Error) Error() string { return string(e) }

0
lib/.gitkeep
View file

94
lib/acl/action.go
View file

@ -1,94 +0,0 @@
package acl
import (
"bytes"
"github.com/nspcc-dev/neofs-api-go/acl"
)
// RequestInfo is an interface of request information needed for extended ACL check.
type RequestInfo interface {
TypedHeaderSource
// Must return the binary representation of request initiator's key.
Key() []byte
// Must return true if request corresponds to operation type.
TypeOf(acl.OperationType) bool
// Must return true if request has passed target.
TargetOf(acl.Target) bool
}
// ExtendedACLChecker is an interface of extended ACL checking tool.
type ExtendedACLChecker interface {
// Must return an action according to the results of applying the ACL table rules to request.
//
// Must return ActionUndefined if it is unable to explicitly calculate the action.
Action(acl.ExtendedACLTable, RequestInfo) acl.ExtendedACLAction
}
type extendedACLChecker struct{}
// NewExtendedACLChecker creates a new extended ACL checking tool and returns ExtendedACLChecker interface.
func NewExtendedACLChecker() ExtendedACLChecker {
return new(extendedACLChecker)
}
// Action returns an action for passed request based on information about it and ACL table.
//
// Returns action of the first suitable table record, or ActionUndefined in the absence thereof.
//
// If passed ExtendedACLTable is nil, ActionUndefined returns.
// If passed RequestInfo is nil, ActionUndefined returns.
func (s extendedACLChecker) Action(table acl.ExtendedACLTable, req RequestInfo) acl.ExtendedACLAction {
if table == nil {
return acl.ActionUndefined
} else if req == nil {
return acl.ActionUndefined
}
for _, record := range table.Records() {
// check type of operation
if !req.TypeOf(record.OperationType()) {
continue
}
// check target
if !targetMatches(req, record.TargetList()) {
continue
}
// check headers
switch MatchFilters(req, record.HeaderFilters()) {
case mResUndefined:
// headers of some type could not be composed => allow
return acl.ActionAllow
case mResMatch:
return record.Action()
}
}
return acl.ActionAllow
}
// returns true if one of ExtendedACLTarget has suitable target OR suitable public key.
func targetMatches(req RequestInfo, list []acl.ExtendedACLTarget) bool {
rKey := req.Key()
for _, target := range list {
// check public key match
for _, key := range target.KeyList() {
if bytes.Equal(key, rKey) {
return true
}
}
// check target group match
if req.TargetOf(target.Target()) {
return true
}
}
return false
}

163
lib/acl/action_test.go
View file

@ -1,163 +0,0 @@
package acl
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/stretchr/testify/require"
)
type testExtendedACLTable struct {
records []acl.ExtendedACLRecord
}
type testRequestInfo struct {
headers []acl.TypedHeader
key []byte
opType acl.OperationType
target acl.Target
}
type testEACLRecord struct {
opType acl.OperationType
filters []acl.HeaderFilter
targets []acl.ExtendedACLTarget
action acl.ExtendedACLAction
}
type testEACLTarget struct {
target acl.Target
keys [][]byte
}
func (s testEACLTarget) Target() acl.Target {
return s.target
}
func (s testEACLTarget) KeyList() [][]byte {
return s.keys
}
func (s testEACLRecord) OperationType() acl.OperationType {
return s.opType
}
func (s testEACLRecord) HeaderFilters() []acl.HeaderFilter {
return s.filters
}
func (s testEACLRecord) TargetList() []acl.ExtendedACLTarget {
return s.targets
}
func (s testEACLRecord) Action() acl.ExtendedACLAction {
return s.action
}
func (s testRequestInfo) HeadersOfType(typ acl.HeaderType) ([]acl.Header, bool) {
res := make([]acl.Header, 0, len(s.headers))
for i := range s.headers {
if s.headers[i].HeaderType() == typ {
res = append(res, s.headers[i])
}
}
return res, true
}
func (s testRequestInfo) Key() []byte {
return s.key
}
func (s testRequestInfo) TypeOf(t acl.OperationType) bool {
return s.opType == t
}
func (s testRequestInfo) TargetOf(t acl.Target) bool {
return s.target == t
}
func (s testExtendedACLTable) Records() []acl.ExtendedACLRecord {
return s.records
}
func TestExtendedACLChecker_Action(t *testing.T) {
s := NewExtendedACLChecker()
// nil ExtendedACLTable
require.Equal(t, acl.ActionUndefined, s.Action(nil, nil))
// create test ExtendedACLTable
table := new(testExtendedACLTable)
// nil RequestInfo
require.Equal(t, acl.ActionUndefined, s.Action(table, nil))
// create test RequestInfo
req := new(testRequestInfo)
// create test ExtendedACLRecord
record := new(testEACLRecord)
table.records = append(table.records, record)
// set different OperationType
record.opType = acl.OperationType(3)
req.opType = record.opType + 1
require.Equal(t, acl.ActionAllow, s.Action(table, req))
// set equal OperationType
req.opType = record.opType
// create test ExtendedACLTarget through group
target := new(testEACLTarget)
record.targets = append(record.targets, target)
// set not matching ExtendedACLTarget
target.target = acl.Target(5)
req.target = target.target + 1
require.Equal(t, acl.ActionAllow, s.Action(table, req))
// set matching ExtendedACLTarget
req.target = target.target
// create test HeaderFilter
fHeader := new(testTypedHeader)
hFilter := &testHeaderFilter{
TypedHeader: fHeader,
}
record.filters = append(record.filters, hFilter)
// create test TypedHeader
header := new(testTypedHeader)
req.headers = append(req.headers, header)
// set not matching values
header.t = hFilter.HeaderType() + 1
require.Equal(t, acl.ActionAllow, s.Action(table, req))
// set matching values
header.k = "key"
header.v = "value"
fHeader.t = header.HeaderType()
fHeader.k = header.Name()
fHeader.v = header.Value()
hFilter.t = acl.StringEqual
// set ExtendedACLAction
record.action = acl.ExtendedACLAction(7)
require.Equal(t, record.action, s.Action(table, req))
// set matching ExtendedACLTarget through key
target.target = req.target + 1
req.key = []byte{1, 2, 3}
target.keys = append(target.keys, req.key)
require.Equal(t, record.action, s.Action(table, req))
}

179
lib/acl/basic.go
View file

@ -1,179 +0,0 @@
package acl
import (
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-node/internal"
)
type (
// BasicChecker is an interface of the basic ACL control tool.
BasicChecker interface {
// Action returns true if request is allowed for this target.
Action(uint32, object.RequestType, acl.Target) (bool, error)
// Bearer returns true if bearer token is allowed for this request.
Bearer(uint32, object.RequestType) (bool, error)
// Extended returns true if extended ACL is allowed for this.
Extended(uint32) bool
// Sticky returns true if sticky bit is set.
Sticky(uint32) bool
}
// BasicACLChecker performs basic ACL check.
BasicACLChecker struct{}
// MaskedBasicACLChecker performs all basic ACL checks, but applying
// mask on ACL first. It is useful, when some bits must be always
// set or unset.
MaskedBasicACLChecker struct {
BasicACLChecker
andMask uint32
orMask uint32
}
nibble struct {
value uint32
}
)
const (
errUnknownRequest = internal.Error("unknown request type")
errUnknownTarget = internal.Error("unknown target type")
)
const (
aclFinalBit = 0x10000000 // 29th bit
aclStickyBit = 0x20000000 // 30th bit
nibbleBBit = 0x1
nibbleOBit = 0x2
nibbleSBit = 0x4
nibbleUBit = 0x8
// DefaultAndFilter is a default AND mask of basic ACL value of container.
DefaultAndFilter = 0xFFFFFFFF
)
var (
nibbleOffset = map[object.RequestType]uint32{
object.RequestGet: 0,
object.RequestHead: 1 * 4,
object.RequestPut: 2 * 4,
object.RequestDelete: 3 * 4,
object.RequestSearch: 4 * 4,
object.RequestRange: 5 * 4,
object.RequestRangeHash: 6 * 4,
}
)
// Action returns true if request is allowed for target.
func (c *BasicACLChecker) Action(rule uint32, req object.RequestType, t acl.Target) (bool, error) {
n, err := fetchNibble(rule, req)
if err != nil {
return false, err
}
switch t {
case acl.Target_User:
return n.U(), nil
case acl.Target_System:
return n.S(), nil
case acl.Target_Others:
return n.O(), nil
default:
return false, errUnknownTarget
}
}
// Bearer returns true if bearer token is allowed to use for this request
// as source of extended ACL.
func (c *BasicACLChecker) Bearer(rule uint32, req object.RequestType) (bool, error) {
n, err := fetchNibble(rule, req)
if err != nil {
return false, err
}
return n.B(), nil
}
// Extended returns true if extended ACL stored in the container are allowed
// to use.
func (c *BasicACLChecker) Extended(rule uint32) bool {
return rule&aclFinalBit != aclFinalBit
}
// Sticky returns true if container is not allowed to store objects with
// owners different from request owner.
func (c *BasicACLChecker) Sticky(rule uint32) bool {
return rule&aclStickyBit == aclStickyBit
}
func fetchNibble(rule uint32, req object.RequestType) (*nibble, error) {
offset, ok := nibbleOffset[req]
if !ok {
return nil, errUnknownRequest
}
return &nibble{value: (rule >> offset) & 0xf}, nil
}
// B returns true if `Bearer` bit set in the nibble.
func (n *nibble) B() bool { return n.value&nibbleBBit == nibbleBBit }
// O returns true if `Others` bit set in the nibble.
func (n *nibble) O() bool { return n.value&nibbleOBit == nibbleOBit }
// S returns true if `System` bit set in the nibble.
func (n *nibble) S() bool { return n.value&nibbleSBit == nibbleSBit }
// U returns true if `User` bit set in the nibble.
func (n *nibble) U() bool { return n.value&nibbleUBit == nibbleUBit }
// NewMaskedBasicACLChecker returns BasicChecker that applies predefined
// bit mask on basic ACL value.
func NewMaskedBasicACLChecker(or, and uint32) BasicChecker {
return MaskedBasicACLChecker{
BasicACLChecker: BasicACLChecker{},
andMask: and,
orMask: or,
}
}
// Action returns true if request is allowed for target.
func (c MaskedBasicACLChecker) Action(rule uint32, req object.RequestType, t acl.Target) (bool, error) {
rule |= c.orMask
rule &= c.andMask
return c.BasicACLChecker.Action(rule, req, t)
}
// Bearer returns true if bearer token is allowed to use for this request
// as source of extended ACL.
func (c MaskedBasicACLChecker) Bearer(rule uint32, req object.RequestType) (bool, error) {
rule |= c.orMask
rule &= c.andMask
return c.BasicACLChecker.Bearer(rule, req)
}
// Extended returns true if extended ACL stored in the container are allowed
// to use.
func (c MaskedBasicACLChecker) Extended(rule uint32) bool {
rule |= c.orMask
rule &= c.andMask
return c.BasicACLChecker.Extended(rule)
}
// Sticky returns true if container is not allowed to store objects with
// owners different from request owner.
func (c MaskedBasicACLChecker) Sticky(rule uint32) bool {
rule |= c.orMask
rule &= c.andMask
return c.BasicACLChecker.Sticky(rule)
}

116
lib/acl/basic_test.go
View file

@ -1,116 +0,0 @@
package acl
import (
"math/bits"
"testing"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/stretchr/testify/require"
)
func TestBasicACLChecker(t *testing.T) {
reqs := []object.RequestType{
object.RequestGet,
object.RequestHead,
object.RequestPut,
object.RequestDelete,
object.RequestSearch,
object.RequestRange,
object.RequestRangeHash,
}
targets := []acl.Target{
acl.Target_Others,
acl.Target_System,
acl.Target_User,
}
checker := new(BasicACLChecker)
t.Run("verb permissions", func(t *testing.T) {
mask := uint32(1)
for i := range reqs {
res, err := checker.Bearer(mask, reqs[i])
require.NoError(t, err)
require.True(t, res)
mask = bits.Reverse32(mask)
res, err = checker.Bearer(mask, reqs[i])
require.NoError(t, err)
require.False(t, res)
mask = bits.Reverse32(mask)
for j := range targets {
mask <<= 1
res, err = checker.Action(mask, reqs[i], targets[j])
require.NoError(t, err)
require.True(t, res)
mask = bits.Reverse32(mask)
res, err = checker.Action(mask, reqs[i], targets[j])
require.NoError(t, err)
require.False(t, res)
mask = bits.Reverse32(mask)
}
mask <<= 1
}
})
t.Run("unknown verb", func(t *testing.T) {
mask := uint32(1)
_, err := checker.Bearer(mask, -1)
require.Error(t, err)
mask = 2
_, err = checker.Action(mask, -1, acl.Target_Others)
require.Error(t, err)
})
t.Run("unknown action", func(t *testing.T) {
mask := uint32(2)
_, err := checker.Action(mask, object.RequestGet, -1)
require.Error(t, err)
})
t.Run("extended acl permission", func(t *testing.T) {
// set F-bit
mask := uint32(0) | aclFinalBit
require.False(t, checker.Extended(mask))
// unset F-bit
mask = bits.Reverse32(mask)
require.True(t, checker.Extended(mask))
})
t.Run("sticky bit permission", func(t *testing.T) {
mask := uint32(0x20000000)
require.True(t, checker.Sticky(mask))
mask = bits.Reverse32(mask)
require.False(t, checker.Sticky(mask))
})
}
// todo: add tests like in basic acl checker
func TestNeoFSMaskedBasicACLChecker(t *testing.T) {
const orFilter = 0x04040444 // this OR filter will be used in neofs-node
checker := NewMaskedBasicACLChecker(orFilter, DefaultAndFilter)
reqs := []object.RequestType{
object.RequestGet,
object.RequestHead,
object.RequestPut,
object.RequestSearch,
object.RequestRangeHash,
}
for i := range reqs {
res, err := checker.Action(0, reqs[i], acl.Target_System)
require.NoError(t, err)
require.True(t, res)
}
}

129
lib/acl/binary.go
View file

@ -1,129 +0,0 @@
package acl
import (
"context"
"encoding/binary"
"io"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-node/internal"
)
// BinaryEACLKey is a binary EACL storage key.
type BinaryEACLKey struct {
cid refs.CID
}
// BinaryEACLValue is a binary EACL storage value.
type BinaryEACLValue struct {
eacl []byte
sig []byte
}
// BinaryExtendedACLSource is an interface of storage of binary extended ACL tables with read access.
type BinaryExtendedACLSource interface {
// Must return binary extended ACL table by key.
GetBinaryEACL(context.Context, BinaryEACLKey) (BinaryEACLValue, error)
}
// BinaryExtendedACLStore is an interface of storage of binary extended ACL tables.
type BinaryExtendedACLStore interface {
BinaryExtendedACLSource
// Must store binary extended ACL table for key.
PutBinaryEACL(context.Context, BinaryEACLKey, BinaryEACLValue) error
}
// ErrNilBinaryExtendedACLStore is returned by function that expect a non-nil
// BinaryExtendedACLStore, but received nil.
const ErrNilBinaryExtendedACLStore = internal.Error("binary extended ACL store is nil")
const sliceLenSize = 4
var eaclEndianness = binary.BigEndian
// CID is a container ID getter.
func (s BinaryEACLKey) CID() refs.CID {
return s.cid
}
// SetCID is a container ID setter.
func (s *BinaryEACLKey) SetCID(v refs.CID) {
s.cid = v
}
// EACL is a binary extended ACL table getter.
func (s BinaryEACLValue) EACL() []byte {
return s.eacl
}
// SetEACL is a binary extended ACL table setter.
func (s *BinaryEACLValue) SetEACL(v []byte) {
s.eacl = v
}
// Signature is an EACL signature getter.
func (s BinaryEACLValue) Signature() []byte {
return s.sig
}
// SetSignature is an EACL signature setter.
func (s *BinaryEACLValue) SetSignature(v []byte) {
s.sig = v
}
// MarshalBinary returns a binary representation of BinaryEACLValue.
func (s BinaryEACLValue) MarshalBinary() ([]byte, error) {
data := make([]byte, sliceLenSize+len(s.eacl)+sliceLenSize+len(s.sig))
off := 0
eaclEndianness.PutUint32(data[off:], uint32(len(s.eacl)))
off += sliceLenSize
off += copy(data[off:], s.eacl)
eaclEndianness.PutUint32(data[off:], uint32(len(s.sig)))
off += sliceLenSize
copy(data[off:], s.sig)
return data, nil
}
// UnmarshalBinary unmarshals BinaryEACLValue from bytes.
func (s *BinaryEACLValue) UnmarshalBinary(data []byte) (err error) {
err = io.ErrUnexpectedEOF
off := 0
if len(data[off:]) < sliceLenSize {
return
}
aclLn := eaclEndianness.Uint32(data[off:])
off += 4
if uint32(len(data[off:])) < aclLn {
return
}
s.eacl = make([]byte, aclLn)
off += copy(s.eacl, data[off:])
if len(data[off:]) < sliceLenSize {
return
}
sigLn := eaclEndianness.Uint32(data[off:])
off += 4
if uint32(len(data[off:])) < sigLn {
return
}
s.sig = make([]byte, sigLn)
copy(s.sig, data[off:])
return nil
}

27
lib/acl/binary_test.go
View file

@ -1,27 +0,0 @@
package acl
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestBinaryEACLValue(t *testing.T) {
s := BinaryEACLValue{}
eacl := []byte{1, 2, 3}
s.SetEACL(eacl)
require.Equal(t, eacl, s.EACL())
sig := []byte{4, 5, 6}
s.SetSignature(sig)
require.Equal(t, sig, s.Signature())
data, err := s.MarshalBinary()
require.NoError(t, err)
s2 := BinaryEACLValue{}
require.NoError(t, s2.UnmarshalBinary(data))
require.Equal(t, s, s2)
}

29
lib/acl/extended.go
View file

@ -1,29 +0,0 @@
package acl
import (
"context"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-api-go/refs"
)
// TypedHeaderSource is a various types of header set interface.
type TypedHeaderSource interface {
// Must return list of Header of particular type.
// Must return false if there is no ability to compose header list.
HeadersOfType(acl.HeaderType) ([]acl.Header, bool)
}
// ExtendedACLSource is an interface of storage of extended ACL tables with read access.
type ExtendedACLSource interface {
// Must return extended ACL table by container ID key.
GetExtendedACLTable(context.Context, refs.CID) (acl.ExtendedACLTable, error)
}
// ExtendedACLStore is an interface of storage of extended ACL tables.
type ExtendedACLStore interface {
ExtendedACLSource
// Must store extended ACL table for container ID key.
PutExtendedACLTable(context.Context, refs.CID, acl.ExtendedACLTable) error
}

234
lib/acl/header.go
View file

@ -1,234 +0,0 @@
package acl
import (
"strconv"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/service"
)
type objectHeaderSource struct {
obj *object.Object
}
type typedHeader struct {
n string
v string
t acl.HeaderType
}
type extendedHeadersWrapper struct {
hdrSrc service.ExtendedHeadersSource
}
type typedExtendedHeader struct {
hdr service.ExtendedHeader
}
func newTypedObjSysHdr(name, value string) acl.TypedHeader {
return &typedHeader{
n: name,
v: value,
t: acl.HdrTypeObjSys,
}
}
// Name is a name field getter.
func (s typedHeader) Name() string {
return s.n
}
// Value is a value field getter.
func (s typedHeader) Value() string {
return s.v
}
// HeaderType is a type field getter.
func (s typedHeader) HeaderType() acl.HeaderType {
return s.t
}
// TypedHeaderSourceFromObject wraps passed object and returns TypedHeaderSource interface.
func TypedHeaderSourceFromObject(obj *object.Object) TypedHeaderSource {
return &objectHeaderSource{
obj: obj,
}
}
// HeaderOfType gathers object headers of passed type and returns Header list.
//
// If value of some header can not be calculated (e.g. nil extended header), it does not appear in list.
//
// Always returns true.
func (s objectHeaderSource) HeadersOfType(typ acl.HeaderType) ([]acl.Header, bool) {
if s.obj == nil {
return nil, true
}
var res []acl.Header
switch typ {
case acl.HdrTypeObjUsr:
objHeaders := s.obj.GetHeaders()
res = make([]acl.Header, 0, len(objHeaders)) // 7 system header fields
for i := range objHeaders {
if h := newTypedObjectExtendedHeader(objHeaders[i]); h != nil {
res = append(res, h)
}
}
case acl.HdrTypeObjSys:
res = make([]acl.Header, 0, 7)
sysHdr := s.obj.GetSystemHeader()
created := sysHdr.GetCreatedAt()
res = append(res,
// ID
newTypedObjSysHdr(
acl.HdrObjSysNameID,
sysHdr.ID.String(),
),
// CID
newTypedObjSysHdr(
acl.HdrObjSysNameCID,
sysHdr.CID.String(),
),
// OwnerID
newTypedObjSysHdr(
acl.HdrObjSysNameOwnerID,
sysHdr.OwnerID.String(),
),
// Version
newTypedObjSysHdr(
acl.HdrObjSysNameVersion,
strconv.FormatUint(sysHdr.GetVersion(), 10),
),
// PayloadLength
newTypedObjSysHdr(
acl.HdrObjSysNamePayloadLength,
strconv.FormatUint(sysHdr.GetPayloadLength(), 10),
),
// CreatedAt.UnitTime
newTypedObjSysHdr(
acl.HdrObjSysNameCreatedUnix,
strconv.FormatUint(uint64(created.GetUnixTime()), 10),
),
// CreatedAt.Epoch
newTypedObjSysHdr(
acl.HdrObjSysNameCreatedEpoch,
strconv.FormatUint(created.GetEpoch(), 10),
),
)
}
return res, true
}
func newTypedObjectExtendedHeader(h object.Header) acl.TypedHeader {
val := h.GetValue()
if val == nil {
return nil
}
res := new(typedHeader)
res.t = acl.HdrTypeObjSys
switch hdr := val.(type) {
case *object.Header_UserHeader:
if hdr.UserHeader == nil {
return nil
}
res.t = acl.HdrTypeObjUsr
res.n = hdr.UserHeader.GetKey()
res.v = hdr.UserHeader.GetValue()
case *object.Header_Link:
if hdr.Link == nil {
return nil
}
switch hdr.Link.GetType() {
case object.Link_Previous:
res.n = acl.HdrObjSysLinkPrev
case object.Link_Next:
res.n = acl.HdrObjSysLinkNext
case object.Link_Child:
res.n = acl.HdrObjSysLinkChild
case object.Link_Parent:
res.n = acl.HdrObjSysLinkPar
case object.Link_StorageGroup:
res.n = acl.HdrObjSysLinkSG
default:
return nil
}
res.v = hdr.Link.ID.String()
default:
return nil
}
return res
}
// TypedHeaderSourceFromExtendedHeaders wraps passed ExtendedHeadersSource and returns TypedHeaderSource interface.
func TypedHeaderSourceFromExtendedHeaders(hdrSrc service.ExtendedHeadersSource) TypedHeaderSource {
return &extendedHeadersWrapper{
hdrSrc: hdrSrc,
}
}
// Name returns the result of Key method.
func (s typedExtendedHeader) Name() string {
return s.hdr.Key()
}
// Value returns the result of Value method.
func (s typedExtendedHeader) Value() string {
return s.hdr.Value()
}
// HeaderType always returns HdrTypeRequest.
func (s typedExtendedHeader) HeaderType() acl.HeaderType {
return acl.HdrTypeRequest
}
// TypedHeaders gathers extended request headers and returns TypedHeader list.
//
// Nil headers are ignored.
//
// Always returns true.
func (s extendedHeadersWrapper) HeadersOfType(typ acl.HeaderType) ([]acl.Header, bool) {
if s.hdrSrc == nil {
return nil, true
}
var res []acl.Header
if typ == acl.HdrTypeRequest {
hs := s.hdrSrc.ExtendedHeaders()
res = make([]acl.Header, 0, len(hs))
for i := range hs {
if hs[i] == nil {
continue
}
res = append(res, &typedExtendedHeader{
hdr: hs[i],
})
}
}
return res, true
}

60
lib/acl/headers_test.go
View file

@ -1,60 +0,0 @@
package acl
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/stretchr/testify/require"
)
func TestNewTypedObjectExtendedHeader(t *testing.T) {
var res acl.TypedHeader
hdr := object.Header{}
// nil value
require.Nil(t, newTypedObjectExtendedHeader(hdr))
// UserHeader
{
key := "key"
val := "val"
hdr.Value = &object.Header_UserHeader{
UserHeader: &object.UserHeader{
Key: key,
Value: val,
},
}
res = newTypedObjectExtendedHeader(hdr)
require.Equal(t, acl.HdrTypeObjUsr, res.HeaderType())
require.Equal(t, key, res.Name())
require.Equal(t, val, res.Value())
}
{ // Link
link := new(object.Link)
link.ID = object.ID{1, 2, 3}
hdr.Value = &object.Header_Link{
Link: link,
}
check := func(lt object.Link_Type, name string) {
link.Type = lt
res = newTypedObjectExtendedHeader(hdr)
require.Equal(t, acl.HdrTypeObjSys, res.HeaderType())
require.Equal(t, name, res.Name())
require.Equal(t, link.ID.String(), res.Value())
}
check(object.Link_Previous, acl.HdrObjSysLinkPrev)
check(object.Link_Next, acl.HdrObjSysLinkNext)
check(object.Link_Parent, acl.HdrObjSysLinkPar)
check(object.Link_Child, acl.HdrObjSysLinkChild)
check(object.Link_StorageGroup, acl.HdrObjSysLinkSG)
}
}

94
lib/acl/match.go
View file

@ -1,94 +0,0 @@
package acl
import (
"github.com/nspcc-dev/neofs-api-go/acl"
)
// Maps MatchType to corresponding function.
// 1st argument of function - header value, 2nd - header filter.
var mMatchFns = map[acl.MatchType]func(acl.Header, acl.Header) bool{
acl.StringEqual: stringEqual,
acl.StringNotEqual: stringNotEqual,
}
const (
mResUndefined = iota
mResMatch
mResMismatch
)
// MatchFilters checks if passed source carry at least one header that satisfies passed filters.
//
// Nil header does not satisfy any filter. Any header does not satisfy nil filter.
//
// Returns mResMismatch if passed TypedHeaderSource is nil.
// Returns mResMatch if passed filters are empty.
//
// If headers for some of the HeaderType could not be composed, mResUndefined returns.
func MatchFilters(src TypedHeaderSource, filters []acl.HeaderFilter) int {
if src == nil {
return mResMismatch
} else if len(filters) == 0 {
return mResMatch
}
matched := 0
for _, filter := range filters {
// prevent NPE
if filter == nil {
continue
}
headers, ok := src.HeadersOfType(filter.HeaderType())
if !ok {
return mResUndefined
}
// get headers of filtering type
for _, header := range headers {
// prevent NPE
if header == nil {
continue
}
// check header name
if header.Name() != filter.Name() {
continue
}
// get match function
matchFn, ok := mMatchFns[filter.MatchType()]
if !ok {
continue
}
// check match
if !matchFn(header, filter) {
continue
}
// increment match counter
matched++
break
}
}
res := mResMismatch
if matched >= len(filters) {
res = mResMatch
}
return res
}
func stringEqual(header, filter acl.Header) bool {
return header.Value() == filter.Value()
}
func stringNotEqual(header, filter acl.Header) bool {
return header.Value() != filter.Value()
}

192
lib/acl/match_test.go
View file

@ -1,192 +0,0 @@
package acl
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/acl"
"github.com/stretchr/testify/require"
)
type testTypedHeader struct {
t acl.HeaderType
k string
v string
}
type testHeaderSrc struct {
hs []acl.TypedHeader
}
type testHeaderFilter struct {
acl.TypedHeader
t acl.MatchType
}
func (s testHeaderFilter) MatchType() acl.MatchType {
return s.t
}
func (s testHeaderSrc) HeadersOfType(typ acl.HeaderType) ([]acl.Header, bool) {
res := make([]acl.Header, 0, len(s.hs))
for i := range s.hs {
if s.hs[i].HeaderType() == typ {
res = append(res, s.hs[i])
}
}
return res, true
}
func (s testTypedHeader) Name() string {
return s.k
}
func (s testTypedHeader) Value() string {
return s.v
}
func (s testTypedHeader) HeaderType() acl.HeaderType {
return s.t
}
func TestMatchFilters(t *testing.T) {
// nil TypedHeaderSource
require.Equal(t, mResMismatch, MatchFilters(nil, nil))
// empty HeaderFilter list
require.Equal(t, mResMatch, MatchFilters(new(testHeaderSrc), nil))
k := "key"
v := "value"
ht := acl.HeaderType(1)
items := []struct {
// list of Key-Value-HeaderType for headers construction
hs []interface{}
// list of Key-Value-HeaderType-MatchType for filters construction
fs []interface{}
exp int
}{
{ // different HeaderType
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v, ht + 1, acl.StringEqual,
},
exp: mResMismatch,
},
{ // different keys
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k + "1", v, ht, acl.StringEqual,
},
exp: mResMismatch,
},
{ // equal values, StringEqual
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v, ht, acl.StringEqual,
},
exp: mResMatch,
},
{ // equal values, StringNotEqual
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v, ht, acl.StringNotEqual,
},
exp: mResMismatch,
},
{ // not equal values, StringEqual
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v + "1", ht, acl.StringEqual,
},
exp: mResMismatch,
},
{ // not equal values, StringNotEqual
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v + "1", ht, acl.StringNotEqual,
},
exp: mResMatch,
},
{ // one header, two filters
hs: []interface{}{
k, v, ht,
},
fs: []interface{}{
k, v + "1", ht, acl.StringNotEqual,
k, v, ht, acl.StringEqual,
},
exp: mResMatch,
},
{ // two headers, one filter
hs: []interface{}{
k, v + "1", ht,
k, v, ht,
},
fs: []interface{}{
k, v, ht, acl.StringEqual,
},
exp: mResMatch,
},
{
hs: []interface{}{
k, v + "1", acl.HdrTypeRequest,
k, v, acl.HdrTypeObjUsr,
},
fs: []interface{}{
k, v, acl.HdrTypeRequest, acl.StringNotEqual,
k, v, acl.HdrTypeObjUsr, acl.StringEqual,
},
exp: mResMatch,
},
}
for _, item := range items {
headers := make([]acl.TypedHeader, 0)
for i := 0; i < len(item.hs); i += 3 {
headers = append(headers, &testTypedHeader{
t: item.hs[i+2].(acl.HeaderType),
k: item.hs[i].(string),
v: item.hs[i+1].(string),
})
}
filters := make([]acl.HeaderFilter, 0)
for i := 0; i < len(item.fs); i += 4 {
filters = append(filters, &testHeaderFilter{
TypedHeader: &testTypedHeader{
t: item.fs[i+2].(acl.HeaderType),
k: item.fs[i].(string),
v: item.fs[i+1].(string),
},
t: item.fs[i+3].(acl.MatchType),
})
}
require.Equal(t,
item.exp,
MatchFilters(
&testHeaderSrc{
hs: headers,
},
filters,
),
)
}
}

24
lib/boot/bootstrap_test.go
View file

@ -1,24 +0,0 @@
package boot
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/bootstrap"
"github.com/stretchr/testify/require"
)
func TestBootstrapPeerParams(t *testing.T) {
s := BootstrapPeerParams{}
nodeInfo := &bootstrap.NodeInfo{
Address: "address",
PubKey: []byte{1, 2, 3},
Options: []string{
"opt1",
"opt2",
},
}
s.SetNodeInfo(nodeInfo)
require.Equal(t, nodeInfo, s.NodeInfo())
}

31
lib/boot/bootstrapper.go
View file

@ -1,31 +0,0 @@
package boot
import (
"github.com/nspcc-dev/neofs-api-go/bootstrap"
"github.com/nspcc-dev/neofs-node/internal"
)
// BootstrapPeerParams is a group of parameters
// for storage node bootstrap.
type BootstrapPeerParams struct {
info *bootstrap.NodeInfo
}
// PeerBootstrapper is an interface of the NeoFS node bootstrap tool.
type PeerBootstrapper interface {
AddPeer(BootstrapPeerParams) error
}
// ErrNilPeerBootstrapper is returned by functions that expect
// a non-nil PeerBootstrapper, but received nil.
const ErrNilPeerBootstrapper = internal.Error("peer bootstrapper is nil")
// SetNodeInfo is a node info setter.
func (s *BootstrapPeerParams) SetNodeInfo(v *bootstrap.NodeInfo) {
s.info = v
}
// NodeInfo is a node info getter.
func (s BootstrapPeerParams) NodeInfo() *bootstrap.NodeInfo {
return s.info
}

46
lib/boot/storage.go
View file

@ -1,46 +0,0 @@
package boot
import (
"context"
"go.uber.org/zap"
)
// StorageBootParams is a group of parameters
// for storage node bootstrap operation.
type StorageBootParams struct {
BootstrapPeerParams
}
// StorageBootController is an entity that performs
// registration of a storage node in NeoFS network.
type StorageBootController struct {
peerBoot PeerBootstrapper
bootPrm StorageBootParams
log *zap.Logger
}
// SetPeerBootstrapper is a PeerBootstrapper setter.
func (s *StorageBootController) SetPeerBootstrapper(v PeerBootstrapper) {
s.peerBoot = v
}
// SetBootParams is a storage node bootstrap parameters setter.
func (s *StorageBootController) SetBootParams(v StorageBootParams) {
s.bootPrm = v
}
// SetLogger is a logging component setter.
func (s *StorageBootController) SetLogger(v *zap.Logger) {
s.log = v
}
// Bootstrap registers storage node in NeoFS system.
func (s StorageBootController) Bootstrap(context.Context) {
// register peer in NeoFS network
if err := s.peerBoot.AddPeer(s.bootPrm.BootstrapPeerParams); err != nil && s.log != nil {
s.log.Error("could not register storage node in network")
}
}

95
lib/buckets/boltdb/methods_test.go
View file

@ -1,95 +0,0 @@
package boltdb
import (
"encoding/binary"
"io/ioutil"
"os"
"strings"
"testing"
"time"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/pkg/errors"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
var config = strings.NewReader(`
storage:
test_bucket:
bucket: boltdb
path: ./temp/storage/test_bucket
perm: 0777
`)
func TestBucket(t *testing.T) {
file, err := ioutil.TempFile("", "test_bolt_db")
require.NoError(t, err)
require.NoError(t, file.Close())
v := viper.New()
require.NoError(t, v.ReadConfig(config))
// -- //
_, err = NewOptions("storage.test_bucket", v)
require.EqualError(t, err, errEmptyPath.Error())
v.SetDefault("storage.test_bucket.path", file.Name())
v.SetDefault("storage.test_bucket.timeout", time.Millisecond*100)
// -- //
opts, err := NewOptions("storage.test_bucket", v)
require.NoError(t, err)
db, err := NewBucket(&opts)
require.NoError(t, err)
require.NotPanics(t, func() { db.Size() })
var (
count = uint64(10)
expected = []byte("test")
)
for i := uint64(0); i < count; i++ {
key := make([]byte, 8)
binary.BigEndian.PutUint64(key, i)
require.False(t, db.Has(key))
val, err := db.Get(key)
require.EqualError(t, errors.Cause(err), core.ErrNotFound.Error())
require.Empty(t, val)
require.NoError(t, db.Set(key, expected))
require.True(t, db.Has(key))
val, err = db.Get(key)
require.NoError(t, err)
require.Equal(t, expected, val)
keys, err := db.List()
require.NoError(t, err)
require.Len(t, keys, 1)
require.Equal(t, key, keys[0])
require.EqualError(t, db.Iterate(nil), core.ErrNilFilterHandler.Error())
items, err := core.ListBucketItems(db, func(_, _ []byte) bool { return true })
require.NoError(t, err)
require.Len(t, items, 1)
require.Equal(t, key, items[0].Key)
require.Equal(t, val, items[0].Val)
require.NoError(t, db.Del(key))
require.False(t, db.Has(key))
val, err = db.Get(key)
require.EqualError(t, errors.Cause(err), core.ErrNotFound.Error())
require.Empty(t, val)
}
require.NoError(t, db.Close())
require.NoError(t, os.RemoveAll(file.Name()))
}

25
lib/buckets/boltdb/plugin/main.go
View file

@ -1,25 +0,0 @@
package main
import (
"github.com/nspcc-dev/neofs-node/lib/buckets/boltdb"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/pkg/errors"
"github.com/spf13/viper"
)
var _ = PrepareBucket
// PrepareBucket is interface method for bucket.
func PrepareBucket(name core.BucketType, v *viper.Viper) (db core.Bucket, err error) {
var opts boltdb.Options
if opts, err = boltdb.NewOptions("storage."+name, v); err != nil {
err = errors.Wrapf(err, "%q: could not prepare options", name)
return
} else if db, err = boltdb.NewBucket(&opts); err != nil {
err = errors.Wrapf(err, "%q: could not prepare bucket", name)
return
}
return
}

64
lib/buckets/init.go
View file

@ -1,64 +0,0 @@
package buckets
import (
"plugin"
"strings"
"github.com/nspcc-dev/neofs-node/lib/buckets/boltdb"
"github.com/nspcc-dev/neofs-node/lib/buckets/fsbucket"
"github.com/nspcc-dev/neofs-node/lib/buckets/inmemory"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/pkg/errors"
"github.com/spf13/viper"
"go.uber.org/zap"
)
const (
// BoltDBBucket is a name of BoltDB bucket.
BoltDBBucket = "boltdb"
// InMemoryBucket is a name RAM bucket.
InMemoryBucket = "in-memory"
// FileSystemBucket is a name of file system bucket.
FileSystemBucket = "fsbucket"
bucketSymbol = "PrepareBucket"
)
// NewBucket is a bucket's constructor.
func NewBucket(name core.BucketType, l *zap.Logger, v *viper.Viper) (core.Bucket, error) {
bucket := v.GetString("storage." + string(name) + ".bucket")
l.Info("initialize bucket",
zap.String("name", string(name)),
zap.String("bucket", bucket))
switch strings.ToLower(bucket) {
case FileSystemBucket:
return fsbucket.NewBucket(name, v)
case InMemoryBucket:
return inmemory.NewBucket(name, v), nil
case BoltDBBucket:
opts, err := boltdb.NewOptions("storage."+name, v)
if err != nil {
return nil, err
}
return boltdb.NewBucket(&opts)
default:
instance, err := plugin.Open(bucket)
if err != nil {
return nil, errors.Wrapf(err, "could not load bucket: `%s`", bucket)
}
sym, err := instance.Lookup(bucketSymbol)
if err != nil {
return nil, errors.Wrapf(err, "could not find bucket signature: `%s`", bucket)
}
return sym.(func(core.BucketType, *viper.Viper) (core.Bucket, error))(name, v)
}
}

60
lib/buckets/inmemory/bucket.go
View file

@ -1,60 +0,0 @@
package inmemory
import (
"sync"
"github.com/mr-tron/base58"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/spf13/viper"
)
type (
bucket struct {
*sync.RWMutex
items map[string][]byte
}
)
const (
defaultCapacity = 100
)
var (
_ core.Bucket = (*bucket)(nil)
// for in usage
_ = NewBucket
)
func stringifyKey(key []byte) string {
return base58.Encode(key)
}
func decodeKey(key string) []byte {
k, err := base58.Decode(key)
if err != nil {
panic(err) // it can fail only for not base58 strings
}
return k
}
func makeCopy(val []byte) []byte {
tmp := make([]byte, len(val))
copy(tmp, val)
return tmp
}
// NewBucket creates new in-memory bucket instance.
func NewBucket(name core.BucketType, v *viper.Viper) core.Bucket {
var capacity int
if capacity = v.GetInt("storage." + string(name) + ".capacity"); capacity <= 0 {
capacity = defaultCapacity
}
return &bucket{
RWMutex: new(sync.RWMutex),
items: make(map[string][]byte, capacity),
}
}

107
lib/buckets/inmemory/methods.go
View file

@ -1,107 +0,0 @@
package inmemory
import (
"unsafe"
"github.com/nspcc-dev/neofs-node/lib/core"
"github.com/pkg/errors"
)
// Get value by key.
func (b *bucket) Get(key []byte) ([]byte, error) {
k := stringifyKey(key)
b.RLock()
val, ok := b.items[k]
result := makeCopy(val)
b.RUnlock()
if !ok {
return nil, errors.Wrapf(core.ErrNotFound, "key=`%s`", k)
}
return result, nil
}
// Set value by key.
func (b *bucket) Set(key, value []byte) error {
k := stringifyKey(key)
b.Lock()
b.items[k] = makeCopy(value)
b.Unlock()
return nil
}
// Del value by key.
func (b *bucket) Del(key []byte) error {
k := stringifyKey(key)
b.Lock()
delete(b.items, k)
b.Unlock()
return nil
}
// Has checks key exists.
func (b *bucket) Has(key []byte) bool {
k := stringifyKey(key)
b.RLock()
_, ok := b.items[k]
b.RUnlock()
return ok
}
// Size size of bucket.
func (b *bucket) Size() int64 {
b.RLock()
// TODO we must replace in future
size := unsafe.Sizeof(b.items)
b.RUnlock()
return int64(size)
}
func (b *bucket) List() ([][]byte, error) {
var result = make([][]byte, 0)
b.RLock()
for key := range b.items {
result = append(result, decodeKey(key))
}
b.RUnlock()
return result, nil
}
// Filter items by closure.
func (b *bucket) Iterate(handler core.FilterHandler) error {
if handler == nil {
return core.ErrNilFilterHandler
}
b.RLock()
for key, val := range b.items {
k, v := decodeKey(key), makeCopy(val)
if !handler(k, v) {
return core.ErrIteratingAborted
}
}
b.RUnlock()
return nil
}
// Close bucket (just empty).
func (b *bucket) Close() error {
b.Lock()
b.items = make(map[string][]byte)
b.Unlock()
return nil
}

15
lib/container/alias.go
View file

@ -1,15 +0,0 @@
package container
import (
"github.com/nspcc-dev/neofs-api-go/container"
"github.com/nspcc-dev/neofs-api-go/refs"
)
// Container is a type alias of Container.
type Container = container.Container
// CID is a type alias of CID.
type CID = refs.CID
// OwnerID is a type alias of OwnerID.
type OwnerID = refs.OwnerID

134
lib/container/storage.go
View file

@ -1,134 +0,0 @@
package container
import (
"context"
)
// GetParams is a group of parameters for container receiving operation.
type GetParams struct {
ctxValue
cidValue
}
// GetResult is a group of values returned by container receiving operation.
type GetResult struct {
cnrValue
}
// PutParams is a group of parameters for container storing operation.
type PutParams struct {
ctxValue
cnrValue
}
// PutResult is a group of values returned by container storing operation.
type PutResult struct {
cidValue
}
// DeleteParams is a group of parameters for container removal operation.
type DeleteParams struct {
ctxValue
cidValue
ownerID OwnerID
}
// DeleteResult is a group of values returned by container removal operation.
type DeleteResult struct{}
// ListParams is a group of parameters for container listing operation.
type ListParams struct {
ctxValue
ownerIDList []OwnerID
}
// ListResult is a group of values returned by container listing operation.
type ListResult struct {
cidList []CID
}
type cnrValue struct {
cnr *Container
}
type cidValue struct {
cid CID
}
type ctxValue struct {
ctx context.Context
}
// Storage is an interface of the storage of NeoFS containers.
type Storage interface {
GetContainer(GetParams) (*GetResult, error)
PutContainer(PutParams) (*PutResult, error)
DeleteContainer(DeleteParams) (*DeleteResult, error)
ListContainers(ListParams) (*ListResult, error)
// TODO: add EACL methods
}
// Context is a context getter.
func (s ctxValue) Context() context.Context {
return s.ctx
}
// SetContext is a context setter.
func (s *ctxValue) SetContext(v context.Context) {
s.ctx = v
}
// CID is a container ID getter.
func (s cidValue) CID() CID {
return s.cid
}
// SetCID is a container ID getter.
func (s *cidValue) SetCID(v CID) {
s.cid = v
}
// Container is a container getter.
func (s cnrValue) Container() *Container {
return s.cnr
}
// SetContainer is a container setter.
func (s *cnrValue) SetContainer(v *Container) {
s.cnr = v
}
// OwnerID is an owner ID getter.
func (s DeleteParams) OwnerID() OwnerID {
return s.ownerID
}
// SetOwnerID is an owner ID setter.
func (s *DeleteParams) SetOwnerID(v OwnerID) {
s.ownerID = v
}
// OwnerIDList is an owner ID list getter.
func (s ListParams) OwnerIDList() []OwnerID {
return s.ownerIDList
}
// SetOwnerIDList is an owner ID list setter.
func (s *ListParams) SetOwnerIDList(v ...OwnerID) {
s.ownerIDList = v
}
// CIDList is a container ID list getter.
func (s ListResult) CIDList() []CID {
return s.cidList
}
// SetCIDList is a container ID list setter.
func (s *ListResult) SetCIDList(v []CID) {
s.cidList = v
}

83
lib/container/storage_test.go
View file

@ -1,83 +0,0 @@
package container
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGetParams(t *testing.T) {
p := new(GetParams)
cid := CID{1, 2, 3}
p.SetCID(cid)
require.Equal(t, cid, p.CID())
}
func TestGetResult(t *testing.T) {
r := new(GetResult)
cnr := &Container{
OwnerID: OwnerID{1, 2, 3},
}
r.SetContainer(cnr)
require.Equal(t, cnr, r.Container())
}
func TestPutParams(t *testing.T) {
p := new(PutParams)
cnr := &Container{
OwnerID: OwnerID{1, 2, 3},
}
p.SetContainer(cnr)
require.Equal(t, cnr, p.Container())
}
func TestPutResult(t *testing.T) {
r := new(PutResult)
cid := CID{1, 2, 3}
r.SetCID(cid)
require.Equal(t, cid, r.CID())
}
func TestDeleteParams(t *testing.T) {
p := new(DeleteParams)
ownerID := OwnerID{1, 2, 3}
p.SetOwnerID(ownerID)
require.Equal(t, ownerID, p.OwnerID())
cid := CID{4, 5, 6}
p.SetCID(cid)
require.Equal(t, cid, p.CID())
}
func TestListParams(t *testing.T) {
p := new(ListParams)
ownerIDList := []OwnerID{
{1, 2, 3},
{4, 5, 6},
}
p.SetOwnerIDList(ownerIDList...)
require.Equal(t, ownerIDList, p.OwnerIDList())
}
func TestListResult(t *testing.T) {
r := new(ListResult)
cidList := []CID{
{1, 2, 3},
{4, 5, 6},
}
r.SetCIDList(cidList)
require.Equal(t, cidList, r.CIDList())
}

94
lib/core/storage.go
View file

@ -1,94 +0,0 @@
package core
import (
"github.com/nspcc-dev/neofs-node/internal"
"github.com/pkg/errors"
)
type (
// BucketType is name of bucket
BucketType string
// FilterHandler where you receive key/val in your closure
FilterHandler func(key, val []byte) bool
// BucketItem used in filter
BucketItem struct {
Key []byte
Val []byte
}
// Bucket is sub-store interface
Bucket interface {
Get(key []byte) ([]byte, error)
Set(key, value []byte) error
Del(key []byte) error
Has(key []byte) bool
Size() int64
List() ([][]byte, error)
Iterate(FilterHandler) error
// Steam can be implemented by badger.Stream, but not for now
// Stream(ctx context.Context, key []byte, cb func(io.ReadWriter) error) error
Close() error
}
// Storage component interface
Storage interface {
GetBucket(name BucketType) (Bucket, error)
Size() int64
Close() error
}
)
const (
// BlobStore is a blob bucket name.
BlobStore BucketType = "blob"
// MetaStore is a meta bucket name.
MetaStore BucketType = "meta"
// SpaceMetricsStore is a space metrics bucket name.
SpaceMetricsStore BucketType = "space-metrics"
)
var (
// ErrNilFilterHandler when FilterHandler is empty
ErrNilFilterHandler = errors.New("handler can't be nil")
// ErrNotFound is returned by key-value storage methods
// that could not find element by key.
ErrNotFound = internal.Error("key not found")
)
// ErrIteratingAborted is returned by storage iterator
// after iteration has been interrupted.
var ErrIteratingAborted = errors.New("iteration aborted")
var errEmptyBucket = errors.New("empty bucket")
func (t BucketType) String() string { return string(t) }
// ListBucketItems performs iteration over Bucket and returns the full list of its items.
func ListBucketItems(b Bucket, h FilterHandler) ([]BucketItem, error) {
if b == nil {
return nil, errEmptyBucket
} else if h == nil {
return nil, ErrNilFilterHandler
}
items := make([]BucketItem, 0)
if err := b.Iterate(func(key, val []byte) bool {
if h(key, val) {
items = append(items, BucketItem{
Key: key,
Val: val,
})
}
return true
}); err != nil {
return nil, err
}
return items, nil
}

65
lib/core/storage_test.go
View file

@ -1,65 +0,0 @@
package core
import (
"crypto/rand"
"testing"
"github.com/stretchr/testify/require"
)
type testBucket struct {
Bucket
items []BucketItem
}
func (s *testBucket) Iterate(f FilterHandler) error {
for i := range s.items {
if !f(s.items[i].Key, s.items[i].Val) {
return ErrIteratingAborted
}
}
return nil
}
func TestListBucketItems(t *testing.T) {
_, err := ListBucketItems(nil, nil)
require.EqualError(t, err, errEmptyBucket.Error())
b := new(testBucket)
_, err = ListBucketItems(b, nil)
require.EqualError(t, err, ErrNilFilterHandler.Error())
var (
count = 10
ln = 10
items = make([]BucketItem, 0, count)
)
for i := 0; i < count; i++ {
items = append(items, BucketItem{
Key: testData(t, ln),
Val: testData(t, ln),
})
}
b.items = items
res, err := ListBucketItems(b, func(key, val []byte) bool { return true })
require.NoError(t, err)
require.Equal(t, items, res)
res, err = ListBucketItems(b, func(key, val []byte) bool { return false })
require.NoError(t, err)
require.Empty(t, res)
}
func testData(t *testing.T, sz int) []byte {
d := make([]byte, sz)
_, err := rand.Read(d)
require.NoError(t, err)
return d
}

392
lib/implementations/acl.go
View file

@ -1,392 +0,0 @@
package implementations
import (
"context"
sc "github.com/nspcc-dev/neo-go/pkg/smartcontract"
libacl "github.com/nspcc-dev/neofs-api-go/acl"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/nspcc-dev/neofs-node/lib/acl"
"github.com/nspcc-dev/neofs-node/lib/blockchain/goclient"
"github.com/nspcc-dev/neofs-node/lib/container"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/pkg/errors"
)
// Consider moving ACLHelper implementation to the ACL library.
type (
// ACLHelper is an interface, that provides useful functions
// for ACL object pre-processor.
ACLHelper interface {
BasicACLGetter
ContainerOwnerChecker
}
// BasicACLGetter helper provides function to return basic ACL value.
BasicACLGetter interface {
GetBasicACL(context.Context, CID) (uint32, error)
}
// ContainerOwnerChecker checks owner of the container.
ContainerOwnerChecker interface {
IsContainerOwner(context.Context, CID, refs.OwnerID) (bool, error)
}
aclHelper struct {
cnr container.Storage
}
)
type binaryEACLSource struct {
binaryStore acl.BinaryExtendedACLSource
}
// StaticContractClient is a wrapper over Neo:Morph client
// that invokes single smart contract methods with fixed fee.
type StaticContractClient struct {
// neo-go client instance
client *goclient.Client
// contract script-hash
scScriptHash util.Uint160
// invocation fee
fee util.Fixed8
}
// MorphContainerContract is a wrapper over StaticContractClient
// for Container contract calls.
type MorphContainerContract struct {
// NeoFS Container smart-contract
containerContract StaticContractClient
// set EACL method name of container contract
eaclSetMethodName string
// get EACL method name of container contract
eaclGetMethodName string
// get container method name of container contract
cnrGetMethodName string
// put container method name of container contract
cnrPutMethodName string
// delete container method name of container contract
cnrDelMethodName string
// list containers method name of container contract
cnrListMethodName string
}
const (
errNewACLHelper = internal.Error("cannot create ACLHelper instance")
)
// GetBasicACL returns basic ACL of the container.
func (h aclHelper) GetBasicACL(ctx context.Context, cid CID) (uint32, error) {
gp := container.GetParams{}
gp.SetContext(ctx)
gp.SetCID(cid)
gResp, err := h.cnr.GetContainer(gp)
if err != nil {
return 0, err
}
return gResp.Container().BasicACL, nil
}
// IsContainerOwner returns true if provided id is an owner container.
func (h aclHelper) IsContainerOwner(ctx context.Context, cid CID, id refs.OwnerID) (bool, error) {
gp := container.GetParams{}
gp.SetContext(ctx)
gp.SetCID(cid)
gResp, err := h.cnr.GetContainer(gp)
if err != nil {
return false, err
}
return gResp.Container().OwnerID.Equal(id), nil
}
// NewACLHelper returns implementation of the ACLHelper interface.
func NewACLHelper(cnr container.Storage) (ACLHelper, error) {
if cnr == nil {
return nil, errNewACLHelper
}
return aclHelper{cnr}, nil
}
// ExtendedACLSourceFromBinary wraps BinaryExtendedACLSource and returns ExtendedACLSource.
//
// If passed BinaryExtendedACLSource is nil, acl.ErrNilBinaryExtendedACLStore returns.
func ExtendedACLSourceFromBinary(v acl.BinaryExtendedACLSource) (acl.ExtendedACLSource, error) {
if v == nil {
return nil, acl.ErrNilBinaryExtendedACLStore
}
return &binaryEACLSource{
binaryStore: v,
}, nil
}
// GetExtendedACLTable receives eACL table in a binary representation from storage,
// unmarshals it and returns ExtendedACLTable interface.
func (s binaryEACLSource) GetExtendedACLTable(ctx context.Context, cid refs.CID) (libacl.ExtendedACLTable, error) {
key := acl.BinaryEACLKey{}
key.SetCID(cid)
val, err := s.binaryStore.GetBinaryEACL(ctx, key)
if err != nil {
return nil, err
}
eacl := val.EACL()
// TODO: verify signature
res := libacl.WrapEACLTable(nil)
return res, res.UnmarshalBinary(eacl)
}
// NewStaticContractClient initializes a new StaticContractClient.
//
// If passed Client is nil, goclient.ErrNilClient returns.
func NewStaticContractClient(client *goclient.Client, scHash util.Uint160, fee util.Fixed8) (StaticContractClient, error) {
res := StaticContractClient{
client: client,
scScriptHash: scHash,
fee: fee,
}
var err error
if client == nil {
err = goclient.ErrNilClient
}
return res, err
}
// Invoke calls Invoke method of goclient with predefined script hash and fee.
// Supported args types are the same as in goclient.
//
// If Client is not initialized, goclient.ErrNilClient returns.
func (s StaticContractClient) Invoke(method string, args ...interface{}) error {
if s.client == nil {
return goclient.ErrNilClient
}
return s.client.Invoke(
s.scScriptHash,
s.fee,
method,
args...,
)
}
// TestInvoke calls TestInvoke method of goclient with predefined script hash.
//
// If Client is not initialized, goclient.ErrNilClient returns.
func (s StaticContractClient) TestInvoke(method string, args ...interface{}) ([]sc.Parameter, error) {
if s.client == nil {
return nil, goclient.ErrNilClient
}
return s.client.TestInvoke(
s.scScriptHash,
method,
args...,
)
}
// SetContainerContractClient is a container contract client setter.
func (s *MorphContainerContract) SetContainerContractClient(v StaticContractClient) {
s.containerContract = v
}
// SetEACLGetMethodName is a container contract Get EACL method name setter.
func (s *MorphContainerContract) SetEACLGetMethodName(v string) {
s.eaclGetMethodName = v
}
// SetEACLSetMethodName is a container contract Set EACL method name setter.
func (s *MorphContainerContract) SetEACLSetMethodName(v string) {
s.eaclSetMethodName = v
}
// SetContainerGetMethodName is a container contract Get method name setter.
func (s *MorphContainerContract) SetContainerGetMethodName(v string) {
s.cnrGetMethodName = v
}
// SetContainerPutMethodName is a container contract Put method name setter.
func (s *MorphContainerContract) SetContainerPutMethodName(v string) {
s.cnrPutMethodName = v
}
// SetContainerDeleteMethodName is a container contract Delete method name setter.
func (s *MorphContainerContract) SetContainerDeleteMethodName(v string) {
s.cnrDelMethodName = v
}
// SetContainerListMethodName is a container contract List method name setter.
func (s *MorphContainerContract) SetContainerListMethodName(v string) {
s.cnrListMethodName = v
}
// GetBinaryEACL performs the test invocation call of GetEACL method of NeoFS Container contract.
func (s *MorphContainerContract) GetBinaryEACL(_ context.Context, key acl.BinaryEACLKey) (acl.BinaryEACLValue, error) {
res := acl.BinaryEACLValue{}
prms, err := s.containerContract.TestInvoke(
s.eaclGetMethodName,
key.CID().Bytes(),
)
if err != nil {
return res, err
} else if ln := len(prms); ln != 1 {
return res, errors.Errorf("unexpected stack parameter count: %d", ln)
}
eacl, err := goclient.BytesFromStackParameter(prms[0])
if err == nil {
res.SetEACL(eacl)
}
return res, err
}
// PutBinaryEACL invokes the call of SetEACL method of NeoFS Container contract.
func (s *MorphContainerContract) PutBinaryEACL(_ context.Context, key acl.BinaryEACLKey, val acl.BinaryEACLValue) error {
return s.containerContract.Invoke(
s.eaclSetMethodName,
key.CID().Bytes(),
val.EACL(),
val.Signature(),
)
}
// GetContainer performs the test invocation call of Get method of NeoFS Container contract.
func (s *MorphContainerContract) GetContainer(p container.GetParams) (*container.GetResult, error) {
prms, err := s.containerContract.TestInvoke(
s.cnrGetMethodName,
p.CID().Bytes(),
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count: %d", ln)
}
cnrBytes, err := goclient.BytesFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get byte array from stack item")
}
cnr := new(container.Container)
if err := cnr.Unmarshal(cnrBytes); err != nil {
return nil, errors.Wrap(err, "could not unmarshal container from bytes")
}
res := new(container.GetResult)
res.SetContainer(cnr)
return res, nil
}
// PutContainer invokes the call of Put method of NeoFS Container contract.
func (s *MorphContainerContract) PutContainer(p container.PutParams) (*container.PutResult, error) {
cnr := p.Container()
cid, err := cnr.ID()
if err != nil {
return nil, errors.Wrap(err, "could not calculate container ID")
}
cnrBytes, err := cnr.Marshal()
if err != nil {
return nil, errors.Wrap(err, "could not marshal container")
}
if err := s.containerContract.Invoke(
s.cnrPutMethodName,
cnr.OwnerID.Bytes(),
cnrBytes,
[]byte{},
); err != nil {
return nil, errors.Wrap(err, "could not invoke contract method")
}
res := new(container.PutResult)
res.SetCID(cid)
return res, nil
}
// DeleteContainer invokes the call of Delete method of NeoFS Container contract.
func (s *MorphContainerContract) DeleteContainer(p container.DeleteParams) (*container.DeleteResult, error) {
if err := s.containerContract.Invoke(
s.cnrDelMethodName,
p.CID().Bytes(),
p.OwnerID().Bytes(),
[]byte{},
); err != nil {
return nil, errors.Wrap(err, "could not invoke contract method")
}
return new(container.DeleteResult), nil
}
// ListContainers performs the test invocation call of Get method of NeoFS Container contract.
//
// If owner ID list in parameters is non-empty, bytes of first owner are attached to call.
func (s *MorphContainerContract) ListContainers(p container.ListParams) (*container.ListResult, error) {
args := make([]interface{}, 0, 1)
if ownerIDList := p.OwnerIDList(); len(ownerIDList) > 0 {
args = append(args, ownerIDList[0].Bytes())
}
prms, err := s.containerContract.TestInvoke(
s.cnrListMethodName,
args...,
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count: %d", ln)
}
prms, err = goclient.ArrayFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get stack item array from stack item")
}
cidList := make([]CID, 0, len(prms))
for i := range prms {
cidBytes, err := goclient.BytesFromStackParameter(prms[i])
if err != nil {
return nil, errors.Wrap(err, "could not get byte array from stack item")
}
cid, err := refs.CIDFromBytes(cidBytes)
if err != nil {
return nil, errors.Wrap(err, "could not get container ID from bytes")
}
cidList = append(cidList, cid)
}
res := new(container.ListResult)
res.SetCIDList(cidList)
return res, nil
}

19
lib/implementations/acl_test.go
View file

@ -1,19 +0,0 @@
package implementations
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestStaticContractClient(t *testing.T) {
s := new(StaticContractClient)
require.NotPanics(t, func() {
_, _ = s.TestInvoke("")
})
require.NotPanics(t, func() {
_ = s.Invoke("")
})
}

141
lib/implementations/balance.go
View file

@ -1,141 +0,0 @@
package implementations
import (
"github.com/nspcc-dev/neo-go/pkg/encoding/address"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-node/lib/blockchain/goclient"
"github.com/pkg/errors"
)
// MorphBalanceContract is a wrapper over NeoFS Balance contract client
// that provides an interface of manipulations with user funds.
type MorphBalanceContract struct {
// NeoFS Balance smart-contract
balanceContract StaticContractClient
// "balance of" method name of balance contract
balanceOfMethodName string
// decimals method name of balance contract
decimalsMethodName string
}
// BalanceOfParams is a structure that groups the parameters
// for NeoFS user balance receiving operation.
type BalanceOfParams struct {
owner refs.OwnerID
}
// BalanceOfResult is a structure that groups the values
// of the result of NeoFS user balance receiving operation.
type BalanceOfResult struct {
amount int64
}
// DecimalsParams is a structure that groups the parameters
// for NeoFS token decimals receiving operation.
type DecimalsParams struct {
}
// DecimalsResult is a structure that groups the values
// of the result of NeoFS token decimals receiving operation.
type DecimalsResult struct {
dec int64
}
// SetBalanceContractClient is a Balance contract client setter.
func (s *MorphBalanceContract) SetBalanceContractClient(v StaticContractClient) {
s.balanceContract = v
}
// SetBalanceOfMethodName is a Balance contract balanceOf method name setter.
func (s *MorphBalanceContract) SetBalanceOfMethodName(v string) {
s.balanceOfMethodName = v
}
// SetDecimalsMethodName is a Balance contract decimals method name setter.
func (s *MorphBalanceContract) SetDecimalsMethodName(v string) {
s.decimalsMethodName = v
}
// BalanceOf performs the test invocation call of balanceOf method of NeoFS Balance contract.
func (s MorphBalanceContract) BalanceOf(p BalanceOfParams) (*BalanceOfResult, error) {
owner := p.OwnerID()
u160, err := address.StringToUint160(owner.String())
if err != nil {
return nil, errors.Wrap(err, "could not convert wallet address to Uint160")
}
prms, err := s.balanceContract.TestInvoke(
s.balanceOfMethodName,
u160.BytesBE(),
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count (balanceOf): %d", ln)
}
amount, err := goclient.IntFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get integer stack item from stack item (amount)")
}
res := new(BalanceOfResult)
res.SetAmount(amount)
return res, nil
}
// Decimals performs the test invocation call of decimals method of NeoFS Balance contract.
func (s MorphBalanceContract) Decimals(DecimalsParams) (*DecimalsResult, error) {
prms, err := s.balanceContract.TestInvoke(
s.decimalsMethodName,
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count (decimals): %d", ln)
}
dec, err := goclient.IntFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get integer stack item from stack item (decimal)")
}
res := new(DecimalsResult)
res.SetDecimals(dec)
return res, nil
}
// SetOwnerID is an owner ID setter.
func (s *BalanceOfParams) SetOwnerID(v refs.OwnerID) {
s.owner = v
}
// OwnerID is an owner ID getter.
func (s BalanceOfParams) OwnerID() refs.OwnerID {
return s.owner
}
// SetAmount is an funds amount setter.
func (s *BalanceOfResult) SetAmount(v int64) {
s.amount = v
}
// Amount is an funds amount getter.
func (s BalanceOfResult) Amount() int64 {
return s.amount
}
// SetDecimals is a decimals setter.
func (s *DecimalsResult) SetDecimals(v int64) {
s.dec = v
}
// Decimals is a decimals getter.
func (s DecimalsResult) Decimals() int64 {
return s.dec
}

35
lib/implementations/balance_test.go
View file

@ -1,35 +0,0 @@
package implementations
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/stretchr/testify/require"
)
func TestBalanceOfParams(t *testing.T) {
s := BalanceOfParams{}
owner := refs.OwnerID{1, 2, 3}
s.SetOwnerID(owner)
require.Equal(t, owner, s.OwnerID())
}
func TestBalanceOfResult(t *testing.T) {
s := BalanceOfResult{}
amount := int64(100)
s.SetAmount(amount)
require.Equal(t, amount, s.Amount())
}
func TestDecimalsResult(t *testing.T) {
s := DecimalsResult{}
dec := int64(100)
s.SetDecimals(dec)
require.Equal(t, dec, s.Decimals())
}

311
lib/implementations/bootstrap.go
View file

@ -1,311 +0,0 @@
package implementations
import (
"github.com/nspcc-dev/neo-go/pkg/smartcontract"
"github.com/nspcc-dev/neofs-api-go/bootstrap"
"github.com/nspcc-dev/neofs-node/lib/blockchain/goclient"
"github.com/nspcc-dev/neofs-node/lib/boot"
"github.com/nspcc-dev/neofs-node/lib/ir"
"github.com/nspcc-dev/neofs-node/lib/netmap"
"github.com/pkg/errors"
)
// MorphNetmapContract is a wrapper over NeoFS Netmap contract client
// that provides an interface of network map manipulations.
type MorphNetmapContract struct {
// NeoFS Netmap smart-contract
netmapContract StaticContractClient
// add peer method name of netmap contract
addPeerMethodName string
// new epoch method name of netmap contract
newEpochMethodName string
// get netmap method name of netmap contract
getNetMapMethodName string
// update state method name of netmap contract
updStateMethodName string
// IR list method name of netmap contract
irListMethodName string
}
// UpdateEpochParams is a structure that groups the parameters
// for NeoFS epoch number updating.
type UpdateEpochParams struct {
epoch uint64
}
// UpdateStateParams is a structure that groups the parameters
// for NeoFS node state updating.
type UpdateStateParams struct {
st NodeState
key []byte
}
// NodeState is a type of node states enumeration.
type NodeState int64
const (
_ NodeState = iota
// StateOffline is an offline node state value.
StateOffline
)
const addPeerFixedArgNumber = 2
const nodeInfoFixedPrmNumber = 3
// SetNetmapContractClient is a Netmap contract client setter.
func (s *MorphNetmapContract) SetNetmapContractClient(v StaticContractClient) {
s.netmapContract = v
}
// SetAddPeerMethodName is a Netmap contract AddPeer method name setter.
func (s *MorphNetmapContract) SetAddPeerMethodName(v string) {
s.addPeerMethodName = v
}
// SetNewEpochMethodName is a Netmap contract NewEpoch method name setter.
func (s *MorphNetmapContract) SetNewEpochMethodName(v string) {
s.newEpochMethodName = v
}
// SetNetMapMethodName is a Netmap contract Netmap method name setter.
func (s *MorphNetmapContract) SetNetMapMethodName(v string) {
s.getNetMapMethodName = v
}
// SetUpdateStateMethodName is a Netmap contract UpdateState method name setter.
func (s *MorphNetmapContract) SetUpdateStateMethodName(v string) {
s.updStateMethodName = v
}
// SetIRListMethodName is a Netmap contract InnerRingList method name setter.
func (s *MorphNetmapContract) SetIRListMethodName(v string) {
s.irListMethodName = v
}
// AddPeer invokes the call of AddPeer method of NeoFS Netmap contract.
func (s *MorphNetmapContract) AddPeer(p boot.BootstrapPeerParams) error {
info := p.NodeInfo()
opts := info.GetOptions()
args := make([]interface{}, 0, addPeerFixedArgNumber+len(opts))
args = append(args,
// Address
[]byte(info.GetAddress()),
// Public key
info.GetPubKey(),
)
// Options
for i := range opts {
args = append(args, []byte(opts[i]))
}
return s.netmapContract.Invoke(
s.addPeerMethodName,
args...,
)
}
// UpdateEpoch invokes the call of NewEpoch method of NeoFS Netmap contract.
func (s *MorphNetmapContract) UpdateEpoch(p UpdateEpochParams) error {
return s.netmapContract.Invoke(
s.newEpochMethodName,
int64(p.Number()), // TODO: do not cast after uint64 type will become supported in client
)
}
// GetNetMap performs the test invocation call of Netmap method of NeoFS Netmap contract.
func (s *MorphNetmapContract) GetNetMap(p netmap.GetParams) (*netmap.GetResult, error) {
prms, err := s.netmapContract.TestInvoke(
s.getNetMapMethodName,
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count (Nodes): %d", ln)
}
prms, err = goclient.ArrayFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get stack item array from stack item (Nodes)")
}
nm := netmap.NewNetmap()
for i := range prms {
nodeInfo, err := nodeInfoFromStackItem(prms[i])
if err != nil {
return nil, errors.Wrapf(err, "could not parse stack item (Node #%d)", i)
}
if err := nm.AddNode(nodeInfo); err != nil {
return nil, errors.Wrapf(err, "could not add node #%d to network map", i)
}
}
res := new(netmap.GetResult)
res.SetNetMap(nm)
return res, nil
}
func nodeInfoFromStackItem(prm smartcontract.Parameter) (*bootstrap.NodeInfo, error) {
prms, err := goclient.ArrayFromStackParameter(prm)
if err != nil {
return nil, errors.Wrapf(err, "could not get stack item array (NodeInfo)")
} else if ln := len(prms); ln != nodeInfoFixedPrmNumber {
return nil, errors.Errorf("unexpected stack item count (NodeInfo): expected %d, has %d", 3, ln)
}
res := new(bootstrap.NodeInfo)
// Address
addrBytes, err := goclient.BytesFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get byte array from stack item (Address)")
}
res.Address = string(addrBytes)
// Public key
res.PubKey, err = goclient.BytesFromStackParameter(prms[1])
if err != nil {
return nil, errors.Wrap(err, "could not get byte array from stack item (Public key)")
}
// Options
prms, err = goclient.ArrayFromStackParameter(prms[2])
if err != nil {
return nil, errors.Wrapf(err, "could not get stack item array (Options)")
}
res.Options = make([]string, 0, len(prms))
for i := range prms {
optBytes, err := goclient.BytesFromStackParameter(prms[i])
if err != nil {
return nil, errors.Wrapf(err, "could not get byte array from stack item (Option #%d)", i)
}
res.Options = append(res.Options, string(optBytes))
}
return res, nil
}
// UpdateState invokes the call of UpdateState method of NeoFS Netmap contract.
func (s *MorphNetmapContract) UpdateState(p UpdateStateParams) error {
return s.netmapContract.Invoke(
s.updStateMethodName,
p.State().Int64(),
p.Key(),
)
}
// GetIRInfo performs the test invocation call of InnerRingList method of NeoFS Netmap contract.
func (s *MorphNetmapContract) GetIRInfo(ir.GetInfoParams) (*ir.GetInfoResult, error) {
prms, err := s.netmapContract.TestInvoke(
s.irListMethodName,
)
if err != nil {
return nil, errors.Wrap(err, "could not perform test invocation")
} else if ln := len(prms); ln != 1 {
return nil, errors.Errorf("unexpected stack item count (Nodes): %d", ln)
}
irInfo, err := irInfoFromStackItem(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get IR info from stack item")
}
res := new(ir.GetInfoResult)
res.SetInfo(*irInfo)
return res, nil
}
func irInfoFromStackItem(prm smartcontract.Parameter) (*ir.Info, error) {
prms, err := goclient.ArrayFromStackParameter(prm)
if err != nil {
return nil, errors.Wrap(err, "could not get stack item array")
}
nodes := make([]ir.Node, 0, len(prms))
for i := range prms {
node, err := irNodeFromStackItem(prms[i])
if err != nil {
return nil, errors.Wrapf(err, "could not get node info from stack item (IRNode #%d)", i)
}
nodes = append(nodes, *node)
}
info := new(ir.Info)
info.SetNodes(nodes)
return info, nil
}
func irNodeFromStackItem(prm smartcontract.Parameter) (*ir.Node, error) {
prms, err := goclient.ArrayFromStackParameter(prm)
if err != nil {
return nil, errors.Wrap(err, "could not get stack item array (IRNode)")
}
// Public key
keyBytes, err := goclient.BytesFromStackParameter(prms[0])
if err != nil {
return nil, errors.Wrap(err, "could not get byte array from stack item (Key)")
}
node := new(ir.Node)
node.SetKey(keyBytes)
return node, nil
}
// SetNumber is an epoch number setter.
func (s *UpdateEpochParams) SetNumber(v uint64) {
s.epoch = v
}
// Number is an epoch number getter.
func (s UpdateEpochParams) Number() uint64 {
return s.epoch
}
// SetState is a state setter.
func (s *UpdateStateParams) SetState(v NodeState) {
s.st = v
}
// State is a state getter.
func (s UpdateStateParams) State() NodeState {
return s.st
}
// SetKey is a public key setter.
func (s *UpdateStateParams) SetKey(v []byte) {
s.key = v
}
// Key is a public key getter.
func (s UpdateStateParams) Key() []byte {
return s.key
}
// Int64 converts NodeState to int64.
func (s NodeState) Int64() int64 {
return int64(s)
}

30
lib/implementations/bootstrap_test.go
View file

@ -1,30 +0,0 @@
package implementations
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestUpdateEpochParams(t *testing.T) {
s := UpdateEpochParams{}
e := uint64(100)
s.SetNumber(e)
require.Equal(t, e, s.Number())
}
func TestUpdateStateParams(t *testing.T) {
s := UpdateStateParams{}
st := NodeState(1)
s.SetState(st)
require.Equal(t, st, s.State())
key := []byte{1, 2, 3}
s.SetKey(key)
require.Equal(t, key, s.Key())
}

7
lib/implementations/epoch.go
View file

@ -1,7 +0,0 @@
package implementations
// EpochReceiver is an interface of the container
// of NeoFS epoch number with read access.
type EpochReceiver interface {
Epoch() uint64
}

41
lib/implementations/reputation.go
View file

@ -1,41 +0,0 @@
package implementations
import (
"github.com/nspcc-dev/neofs-node/lib/peers"
)
// MorphReputationContract is a wrapper over NeoFS Reputation contract client
// that provides an interface of the storage of global trust values.
type MorphReputationContract struct {
// NeoFS Reputation smart-contract
repContract StaticContractClient
// put method name of reputation contract
putMethodName string
// list method name of reputation contract
listMethodName string
// public key storage
pkStore peers.PublicKeyStore
}
// SetReputationContractClient is a Reputation contract client setter.
func (s *MorphReputationContract) SetReputationContractClient(v StaticContractClient) {
s.repContract = v
}
// SetPublicKeyStore is a public key store setter.
func (s *MorphReputationContract) SetPublicKeyStore(v peers.PublicKeyStore) {
s.pkStore = v
}
// SetPutMethodName is a Reputation contract Put method name setter.
func (s *MorphReputationContract) SetPutMethodName(v string) {
s.putMethodName = v
}
// SetListMethodName is a Reputation contract List method name setter.
func (s *MorphReputationContract) SetListMethodName(v string) {
s.listMethodName = v
}

17
lib/ir/info.go
View file

@ -1,17 +0,0 @@
package ir
// Info is a structure that groups the information
// about inner ring.
type Info struct {
nodes []Node
}
// SetNodes is an IR node list setter.
func (s *Info) SetNodes(v []Node) {
s.nodes = v
}
// Nodes is an IR node list getter.
func (s Info) Nodes() []Node {
return s.nodes
}

25
lib/ir/info_test.go
View file

@ -1,25 +0,0 @@
package ir
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestInfo(t *testing.T) {
s := Info{}
n1 := Node{}
n1.SetKey([]byte{1, 2, 3})
n2 := Node{}
n2.SetKey([]byte{4, 5, 6})
nodes := []Node{
n1,
n2,
}
s.SetNodes(nodes)
require.Equal(t, nodes, s.Nodes())
}

17
lib/ir/node.go
View file

@ -1,17 +0,0 @@
package ir
// Node is a structure that groups
// the information about IR node.
type Node struct {
key []byte
}
// SetKey is an IR node public key setter.
func (s *Node) SetKey(v []byte) {
s.key = v
}
// Key is an IR node public key getter.
func (s Node) Key() []byte {
return s.key
}

16
lib/ir/node_test.go
View file

@ -1,16 +0,0 @@
package ir
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestNode(t *testing.T) {
s := Node{}
key := []byte{1, 2, 3}
s.SetKey(key)
require.Equal(t, key, s.Key())
}

94
lib/ir/storage.go
View file

@ -1,94 +0,0 @@
package ir
import (
"bytes"
crypto "github.com/nspcc-dev/neofs-crypto"
"github.com/nspcc-dev/neofs-node/internal"
"github.com/pkg/errors"
)
// Storage is an interface of the storage of info about NeoFS IR.
type Storage interface {
GetIRInfo(GetInfoParams) (*GetInfoResult, error)
}
// GetInfoParams is a structure that groups the parameters
// for IR info receiving operation.
type GetInfoParams struct {
}
// GetInfoResult is a structure that groups
// values returned by IR info receiving operation.
type GetInfoResult struct {
info Info
}
// ErrNilStorage is returned by functions that expect
// a non-nil Storage, but received nil.
const ErrNilStorage = internal.Error("inner ring storage is nil")
// SetInfo is an IR info setter.
func (s *GetInfoResult) SetInfo(v Info) {
s.info = v
}
// Info is an IR info getter.
func (s GetInfoResult) Info() Info {
return s.info
}
// BinaryKeyList returns the list of binary public key of IR nodes.
//
// If passed Storage is nil, ErrNilStorage returns.
func BinaryKeyList(storage Storage) ([][]byte, error) {
if storage == nil {
return nil, ErrNilStorage
}
// get IR info
getRes, err := storage.GetIRInfo(GetInfoParams{})
if err != nil {
return nil, errors.Wrap(
err,
"could not get information about IR",
)
}
nodes := getRes.Info().Nodes()
keys := make([][]byte, 0, len(nodes))
for i := range nodes {
keys = append(keys, nodes[i].Key())
}
return keys, nil
}
// IsInnerRingKey checks if the passed argument is the
// key of one of IR nodes.
//
// Uses BinaryKeyList function to receive the key list of IR nodes internally.
//
// If passed key slice is empty, crypto.ErrEmptyPublicKey returns immediately.
func IsInnerRingKey(storage Storage, key []byte) (bool, error) {
// check key emptiness
// TODO: summarize the void check to a full IR key-format check.
if len(key) == 0 {
return false, crypto.ErrEmptyPublicKey
}
irKeys, err := BinaryKeyList(storage)
if err != nil {
return false, err
}
for i := range irKeys {
if bytes.Equal(irKeys[i], key) {
return true, nil
}
}
return false, nil
}

101
lib/ir/storage_test.go
View file

@ -1,101 +0,0 @@
package ir
import (
"testing"
crypto "github.com/nspcc-dev/neofs-crypto"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
)
type testInfoReceiver struct {
keys [][]byte
err error
}
func (s testInfoReceiver) GetIRInfo(GetInfoParams) (*GetInfoResult, error) {
if s.err != nil {
return nil, s.err
}
nodes := make([]Node, 0, len(s.keys))
for i := range s.keys {
node := Node{}
node.SetKey(s.keys[i])
nodes = append(nodes, node)
}
info := Info{}
info.SetNodes(nodes)
res := new(GetInfoResult)
res.SetInfo(info)
return res, nil
}
func (s *testInfoReceiver) addKey(key []byte) {
s.keys = append(s.keys, key)
}
func TestGetInfoResult(t *testing.T) {
s := GetInfoResult{}
info := Info{}
n := Node{}
n.SetKey([]byte{1, 2, 3})
info.SetNodes([]Node{
n,
})
s.SetInfo(info)
require.Equal(t, info, s.Info())
}
func TestIsInnerRingKey(t *testing.T) {
var (
res bool
err error
s = new(testInfoReceiver)
)
// empty public key
res, err = IsInnerRingKey(nil, nil)
require.EqualError(t, err, crypto.ErrEmptyPublicKey.Error())
key := []byte{1, 2, 3}
// nil Storage
res, err = IsInnerRingKey(nil, key)
require.EqualError(t, err, ErrNilStorage.Error())
// force Storage to return an error
s.err = errors.New("some error")
// Storage error
res, err = IsInnerRingKey(s, key)
require.EqualError(t, errors.Cause(err), s.err.Error())
// reset Storage error
s.err = nil
// IR keys don't contain key
s.addKey(append(key, 1))
res, err = IsInnerRingKey(s, key)
require.NoError(t, err)
require.False(t, res)
// IR keys contain key
s.addKey(key)
res, err = IsInnerRingKey(s, key)
require.NoError(t, err)
require.True(t, res)
}

392
lib/netmap/netmap.go
View file

@ -1,392 +0,0 @@
package netmap
import (
"crypto/sha256"
"encoding/json"
"reflect"
"sort"
"sync"
"github.com/nspcc-dev/neofs-api-go/bootstrap"
"github.com/nspcc-dev/netmap"
"github.com/pkg/errors"
"github.com/spaolacci/murmur3"
)
type (
// Bucket is an alias for github.com/nspcc-dev/netmap.Bucket
Bucket = netmap.Bucket
// SFGroup is an alias for github.com/nspcc-dev/netmap.SFGroup
SFGroup = netmap.SFGroup
// Select is an alias for github.com/nspcc-dev/netmap.Select
Select = netmap.Select
// Filter is an alias for github.com/nspcc-dev/netmap.Filter
Filter = netmap.Filter
// SimpleFilter is an alias for github.com/nspcc-dev/netmap.Filter
SimpleFilter = netmap.SimpleFilter
// PlacementRule is an alias for github.com/nspcc-dev/netmap.Filter
PlacementRule = netmap.PlacementRule
// NetMap is a general network map structure for NeoFS
NetMap struct {
mu *sync.RWMutex
root Bucket
items Nodes
}
// Nodes is an alias for slice of NodeInfo which is structure that describes every host
Nodes []bootstrap.NodeInfo
)
const (
// Separator separates key:value pairs in string representation of options.
Separator = netmap.Separator
// NodesBucket is the name for optionless bucket containing only nodes.
NodesBucket = netmap.NodesBucket
)
var (
// FilterIn returns filter, which checks if value is in specified list.
FilterIn = netmap.FilterIn
// FilterNotIn returns filter, which checks if value is not in specified list.
FilterNotIn = netmap.FilterNotIn
// FilterOR returns OR combination of filters.
FilterOR = netmap.FilterOR
// FilterAND returns AND combination of filters.
FilterAND = netmap.FilterAND
// FilterEQ returns filter, which checks if value is equal to v.
FilterEQ = netmap.FilterEQ
// FilterNE returns filter, which checks if value is not equal to v.
FilterNE = netmap.FilterNE
// FilterGT returns filter, which checks if value is greater than v.
FilterGT = netmap.FilterGT
// FilterGE returns filter, which checks if value is greater or equal than v.
FilterGE = netmap.FilterGE
// FilterLT returns filter, which checks if value is less than v.
FilterLT = netmap.FilterLT
// FilterLE returns filter, which checks if value is less or equal than v.
FilterLE = netmap.FilterLE
)
var errNetMapsConflict = errors.New("netmaps are in conflict")
// Copy creates new slice of copied nodes.
func (n Nodes) Copy() Nodes {
res := make(Nodes, len(n))
for i := range n {
res[i].Address = n[i].Address
res[i].Status = n[i].Status
if n[i].PubKey != nil {
res[i].PubKey = make([]byte, len(n[i].PubKey))
copy(res[i].PubKey, n[i].PubKey)
}
if n[i].Options != nil {
res[i].Options = make([]string, len(n[i].Options))
copy(res[i].Options, n[i].Options)
}
}
return res
}
// NewNetmap is an constructor.
func NewNetmap() *NetMap {
return &NetMap{
items: make([]bootstrap.NodeInfo, 0),
mu: new(sync.RWMutex),
}
}
// Equals return whether two netmap are identical.
func (n *NetMap) Equals(nm *NetMap) bool {
n.mu.RLock()
defer n.mu.RUnlock()
return len(n.items) == len(nm.items) &&
n.root.Equals(nm.root) &&
reflect.DeepEqual(n.items, nm.items)
}
// Root returns netmap root-bucket.
func (n *NetMap) Root() *Bucket {
n.mu.RLock()
cp := n.root.Copy()
n.mu.RUnlock()
return &cp
}
// Copy creates and returns full copy of target netmap.
func (n *NetMap) Copy() *NetMap {
n.mu.RLock()
defer n.mu.RUnlock()
nm := NewNetmap()
nm.items = n.items.Copy()
nm.root = n.root.Copy()
return nm
}
type hashedItem struct {
h uint32
info *bootstrap.NodeInfo
}
// Normalise reorders netmap items into some canonical order.
func (n *NetMap) Normalise() *NetMap {
nm := NewNetmap()
items := n.items.Copy()
if len(items) == 0 {
return nm
}
itemsH := make([]hashedItem, len(n.items))
for i := range itemsH {
itemsH[i].h = murmur3.Sum32(n.items[i].PubKey)
itemsH[i].info = &items[i]
}
sort.Slice(itemsH, func(i, j int) bool {
if itemsH[i].h == itemsH[j].h {
return itemsH[i].info.Address < itemsH[j].info.Address
}
return itemsH[i].h < itemsH[j].h
})
lastHash := ^itemsH[0].h
lastAddr := ""
for i := range itemsH {
if itemsH[i].h != lastHash || itemsH[i].info.Address != lastAddr {
_ = nm.AddNode(itemsH[i].info)
lastHash = itemsH[i].h
}
}
return nm
}
// Hash returns hash of n.
func (n *NetMap) Hash() (sum [32]byte) {
items := n.Normalise().Items()
w := sha256.New()
for i := range items {
data, _ := items[i].Marshal()
_, _ = w.Write(data)
}
s := w.Sum(nil)
copy(sum[:], s)
return
}
// InheritWeights calculates average capacity and minimal price, then provides buckets with IQR weight.
func (n *NetMap) InheritWeights() *NetMap {
nm := n.Copy()
// find average capacity in the network map
meanCap := nm.root.Traverse(netmap.NewMeanAgg(), netmap.CapWeightFunc).Compute()
capNorm := netmap.NewSigmoidNorm(meanCap)
// find minimal price in the network map
minPrice := nm.root.Traverse(netmap.NewMinAgg(), netmap.PriceWeightFunc).Compute()
priceNorm := netmap.NewReverseMinNorm(minPrice)
// provide all buckets with
wf := netmap.NewWeightFunc(capNorm, priceNorm)
meanAF := netmap.AggregatorFactory{New: netmap.NewMeanIQRAgg}
nm.root.TraverseTree(meanAF, wf)
return nm
}
// Merge checks if merge is possible and then add new elements from given netmap.
func (n *NetMap) Merge(n1 *NetMap) error {
n.mu.Lock()
defer n.mu.Unlock()
var (
tr = make(map[uint32]netmap.Node, len(n1.items))
items = n.items
)
loop:
for j := range n1.items {
for i := range n.items {
if n.items[i].Equals(n1.items[j]) {
tr[uint32(j)] = netmap.Node{
N: uint32(i),
C: n.items[i].Capacity(),
P: n.items[i].Price(),
}
continue loop
}
}
tr[uint32(j)] = netmap.Node{
N: uint32(len(items)),
C: n1.items[j].Capacity(),
P: n1.items[j].Price(),
}
items = append(items, n1.items[j])
}
root := n1.root.UpdateIndices(tr)
if n.root.CheckConflicts(root) {
return errNetMapsConflict
}
n.items = items
n.root.Merge(root)
return nil
}
// FindGraph finds sub-graph filtered by given SFGroup.
func (n *NetMap) FindGraph(pivot []byte, ss ...SFGroup) (c *Bucket) {
n.mu.RLock()
defer n.mu.RUnlock()
return n.root.FindGraph(pivot, ss...)
}
// FindNodes finds sub-graph filtered by given SFGroup and returns all sub-graph items.
func (n *NetMap) FindNodes(pivot []byte, ss ...SFGroup) (nodes []uint32) {
n.mu.RLock()
defer n.mu.RUnlock()
return n.root.FindNodes(pivot, ss...).Nodes()
}
// Items return slice of all NodeInfo in netmap.
func (n *NetMap) Items() []bootstrap.NodeInfo {
n.mu.RLock()
defer n.mu.RUnlock()
return n.items
}
// ItemsCopy return copied slice of all NodeInfo in netmap (is it useful?).
func (n *NetMap) ItemsCopy() Nodes {
n.mu.RLock()
defer n.mu.RUnlock()
return n.items.Copy()
}
// Add adds node with given address and given options.
func (n *NetMap) Add(addr string, pk []byte, st bootstrap.NodeStatus, opts ...string) error {
return n.AddNode(&bootstrap.NodeInfo{Address: addr, PubKey: pk, Status: st, Options: opts})
}
// Update replaces netmap with given netmap.
func (n *NetMap) Update(nxt *NetMap) {
n.mu.Lock()
defer n.mu.Unlock()
n.root = nxt.root
n.items = nxt.items
}
// GetMaxSelection returns 'maximal container' -- subgraph which contains
// any other subgraph satisfying specified selects and filters.
func (n *NetMap) GetMaxSelection(ss []Select, fs []Filter) (r *Bucket) {
return n.root.GetMaxSelection(netmap.SFGroup{Selectors: ss, Filters: fs})
}
// AddNode adds to exited or new node slice of given options.
func (n *NetMap) AddNode(nodeInfo *bootstrap.NodeInfo, opts ...string) error {
n.mu.Lock()
defer n.mu.Unlock()
info := *nodeInfo
info.Options = append(info.Options, opts...)
num := -1
// looking for existed node info item
for i := range n.items {
if n.items[i].Equals(info) {
num = i
break
}
}
// if item is not existed - add it
if num < 0 {
num = len(n.items)
n.items = append(n.items, info)
}
return n.root.AddStrawNode(netmap.Node{
N: uint32(num),
C: n.items[num].Capacity(),
P: n.items[num].Price(),
}, info.Options...)
}
// GetNodesByOption returns slice of NodeInfo that has given option.
func (n *NetMap) GetNodesByOption(opts ...string) []bootstrap.NodeInfo {
n.mu.RLock()
defer n.mu.RUnlock()
ns := n.root.GetNodesByOption(opts...)
nodes := make([]bootstrap.NodeInfo, 0, len(ns))
for _, info := range ns {
nodes = append(nodes, n.items[info.N])
}
return nodes
}
// MarshalJSON custom marshaller.
func (n *NetMap) MarshalJSON() ([]byte, error) {
n.mu.RLock()
defer n.mu.RUnlock()
return json.Marshal(n.items)
}
// UnmarshalJSON custom unmarshaller.
func (n *NetMap) UnmarshalJSON(data []byte) error {
var (
nm = NewNetmap()
items []bootstrap.NodeInfo
)
if err := json.Unmarshal(data, &items); err != nil {
return err
}
for i := range items {
if err := nm.Add(items[i].Address, items[i].PubKey, items[i].Status, items[i].Options...); err != nil {
return err
}
}
if n.mu == nil {
n.mu = new(sync.RWMutex)
}
n.mu.Lock()
n.root = nm.root
n.items = nm.items
n.mu.Unlock()
return nil
}
// Size returns number of nodes in network map.
func (n *NetMap) Size() int {
n.mu.RLock()
defer n.mu.RUnlock()
return len(n.items)
}

261
lib/netmap/netmap_test.go
View file

@ -1,261 +0,0 @@
package netmap
import (
"bytes"
"encoding/json"
"math/rand"
"sync"
"testing"
"time"
"github.com/nspcc-dev/neofs-api-go/bootstrap"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/netmap"
"github.com/stretchr/testify/require"
)
func TestNetMap_DataRace(t *testing.T) {
var (
nm = NewNetmap()
wg = new(sync.WaitGroup)
nodes = []bootstrap.NodeInfo{
{Address: "SPB1", Options: []string{"/Location:Europe/Country:USA"}},
{Address: "SPB2", Options: []string{"/Location:Europe/Country:Italy"}},
{Address: "MSK1", Options: []string{"/Location:Europe/Country:Germany"}},
{Address: "MSK2", Options: []string{"/Location:Europe/Country:Russia"}},
}
)
wg.Add(10)
for i := 0; i < 10; i++ {
go func(n int) {
for _, node := range nodes {
require.NoError(t, nm.Add(node.Address, node.PubKey, 0, node.Options...))
// t.Logf("%02d: add node %q", n, node.Address)
}
wg.Done()
}(i)
}
wg.Add(3 * 10)
for i := 0; i < 10; i++ {
go func(n int) {
nm.Copy()
// t.Logf("%02d: Copy", n)
wg.Done()
}(i)
go func(n int) {
nm.Items()
// t.Logf("%02d: Items", n)
wg.Done()
}(i)
go func(n int) {
nm.Root()
// t.Logf("%02d: Root", n)
wg.Done()
}(i)
}
wg.Wait()
}
func TestNetMapSuite(t *testing.T) {
var (
err error
nm1 = NewNetmap()
nodes = []bootstrap.NodeInfo{
{Address: "SPB1", Options: []string{"/Location:Europe/Country:USA"}, Status: 1},
{Address: "SPB2", Options: []string{"/Location:Europe/Country:Italy"}, Status: 2},
{Address: "MSK1", Options: []string{"/Location:Europe/Country:Germany"}, Status: 3},
{Address: "MSK2", Options: []string{"/Location:Europe/Country:Russia"}, Status: 4},
}
)
for _, node := range nodes {
err = nm1.Add(node.Address, nil, node.Status, node.Options...)
require.NoError(t, err)
}
t.Run("copy should work like expected", func(t *testing.T) {
nm2 := nm1.Copy()
require.Equal(t, nm1.root, nm2.root)
require.Equal(t, nm1.items, nm2.items)
})
t.Run("add node should not ignore options", func(t *testing.T) {
items := nm1.ItemsCopy()
nm2 := NewNetmap()
err = nm2.AddNode(&items[0], "/New/Option")
require.NoError(t, err)
require.Len(t, nm2.items, 1)
require.Equal(t, append(items[0].Options, "/New/Option"), nm2.items[0].Options)
})
t.Run("copyItems should work like expected", func(t *testing.T) {
require.Equal(t, nm1.items, nm1.ItemsCopy())
})
t.Run("marshal / unmarshal should be identical on same data", func(t *testing.T) {
var nm2 *NetMap
want, err := json.Marshal(nodes)
require.NoError(t, err)
actual, err := json.Marshal(nm1)
require.NoError(t, err)
require.Equal(t, want, actual)
err = json.Unmarshal(actual, &nm2)
require.NoError(t, err)
require.Equal(t, nm1.root, nm2.root)
require.Equal(t, nm1.items, nm2.items)
})
t.Run("unmarshal should override existing data", func(t *testing.T) {
var nm2 *NetMap
want, err := json.Marshal(nodes)
require.NoError(t, err)
actual, err := json.Marshal(nm1)
require.NoError(t, err)
require.Equal(t, want, actual)
nm2 = nm1.Copy()
err = nm2.Add("SOMEADDR", nil, 0, "/Location:Europe/Country:USA")
require.NoError(t, err)
err = json.Unmarshal(actual, &nm2)
require.NoError(t, err)
require.Equal(t, nm1.root, nm2.root)
require.Equal(t, nm1.items, nm2.items)
})
t.Run("unmarshal should fail on bad data", func(t *testing.T) {
var nm2 *NetMap
require.Error(t, json.Unmarshal([]byte(`"some bad data"`), &nm2))
})
t.Run("unmarshal should fail on add nodes", func(t *testing.T) {
var nm2 *NetMap
require.Error(t, json.Unmarshal([]byte(`[{"address": "SPB1","options":["1-2-3-4"]}]`), &nm2))
})
t.Run("merge two netmaps", func(t *testing.T) {
newNodes := []bootstrap.NodeInfo{
{Address: "SPB3", Options: []string{"/Location:Europe/Country:France"}},
}
nm2 := NewNetmap()
for _, node := range newNodes {
err = nm2.Add(node.Address, nil, 0, node.Options...)
require.NoError(t, err)
}
err = nm2.Merge(nm1)
require.NoError(t, err)
require.Len(t, nm2.items, len(nodes)+len(newNodes))
ns := nm2.FindNodes([]byte("pivot"), netmap.SFGroup{
Filters: []Filter{{Key: "Country", F: FilterEQ("Germany")}},
Selectors: []Select{{Count: 1, Key: NodesBucket}},
})
require.Len(t, ns, 1)
})
t.Run("weighted netmaps", func(t *testing.T) {
strawNodes := []bootstrap.NodeInfo{
{Address: "SPB2", Options: []string{"/Location:Europe/Country:Italy", "/Capacity:10", "/Price:100"}},
{Address: "MSK1", Options: []string{"/Location:Europe/Country:Germany", "/Capacity:10", "/Price:1"}},
{Address: "MSK2", Options: []string{"/Location:Europe/Country:Russia", "/Capacity:5", "/Price:10"}},
{Address: "SPB1", Options: []string{"/Location:Europe/Country:France", "/Capacity:20", "/Price:2"}},
}
nm2 := NewNetmap()
for _, node := range strawNodes {
err = nm2.Add(node.Address, nil, 0, node.Options...)
require.NoError(t, err)
}
ns1 := nm1.FindNodes([]byte("pivot"), netmap.SFGroup{
Selectors: []Select{{Count: 2, Key: NodesBucket}},
})
require.Len(t, ns1, 2)
ns2 := nm2.FindNodes([]byte("pivot"), netmap.SFGroup{
Selectors: []Select{{Count: 2, Key: NodesBucket}},
})
require.Len(t, ns2, 2)
require.NotEqual(t, ns1, ns2)
require.Equal(t, []uint32{1, 3}, ns2)
})
}
func TestNetMap_Normalise(t *testing.T) {
const testCount = 5
nodes := []bootstrap.NodeInfo{
{Address: "SPB2", PubKey: []byte{4}, Options: []string{"/Location:Europe/Country:Italy", "/Capacity:10", "/Price:100"}},
{Address: "MSK1", PubKey: []byte{2}, Options: []string{"/Location:Europe/Country:Germany", "/Capacity:10", "/Price:1"}},
{Address: "MSK2", PubKey: []byte{3}, Options: []string{"/Location:Europe/Country:Russia", "/Capacity:5", "/Price:10"}},
{Address: "SPB1", PubKey: []byte{1}, Options: []string{"/Location:Europe/Country:France", "/Capacity:20", "/Price:2"}},
}
add := func(nm *NetMap, indices ...int) {
for _, i := range indices {
err := nm.Add(nodes[i].Address, nodes[i].PubKey, 0, nodes[i].Options...)
require.NoError(t, err)
}
}
indices := []int{0, 1, 2, 3}
nm1 := NewNetmap()
add(nm1, indices...)
norm := nm1.Normalise()
for i := 0; i < testCount; i++ {
rand.Seed(time.Now().UnixNano())
rand.Shuffle(len(indices), func(i, j int) { indices[i], indices[j] = indices[j], indices[i] })
nm := NewNetmap()
add(nm, indices...)
require.Equal(t, norm, nm.Normalise())
}
t.Run("normalise removes duplicates", func(t *testing.T) {
before := NewNetmap()
add(before, indices...)
before.items = append(before.items, before.items...)
nm := before.Normalise()
require.Len(t, nm.items, len(indices))
loop:
for i := range nodes {
for j := range nm.items {
if bytes.Equal(nm.items[j].PubKey, nodes[i].PubKey) {
continue loop
}
}
require.Fail(t, "normalized netmap does not contain '%s' node", nodes[i].Address)
}
})
}
func TestNodeInfo_Price(t *testing.T) {
var info bootstrap.NodeInfo
// too small value
info = bootstrap.NodeInfo{Options: []string{"/Price:0.01048575"}}
require.Equal(t, uint64(0), info.Price())
// min value
info = bootstrap.NodeInfo{Options: []string{"/Price:0.01048576"}}
require.Equal(t, uint64(1), info.Price())
// big value
info = bootstrap.NodeInfo{Options: []string{"/Price:1000000000.666"}}
require.Equal(t, uint64(1000000000.666*1e8/object.UnitsMB), info.Price())
}

27
lib/netmap/storage.go
View file

@ -1,27 +0,0 @@
package netmap
// GetParams is a group of parameters
// for network map receiving operation.
type GetParams struct {
}
// GetResult is a group of values
// returned by container receiving operation.
type GetResult struct {
nm *NetMap
}
// Storage is an interface of the storage of NeoFS network map.
type Storage interface {
GetNetMap(GetParams) (*GetResult, error)
}
// NetMap is a network map getter.
func (s GetResult) NetMap() *NetMap {
return s.nm
}
// SetNetMap is a network map setter.
func (s *GetResult) SetNetMap(v *NetMap) {
s.nm = v
}

23
lib/netmap/storage_test.go
View file

@ -1,23 +0,0 @@
package netmap
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/bootstrap"
"github.com/stretchr/testify/require"
)
func TestGetResult(t *testing.T) {
s := GetResult{}
nm := NewNetmap()
require.NoError(t,
nm.AddNode(&bootstrap.NodeInfo{
Address: "address",
PubKey: []byte{1, 2, 3},
}),
)
s.SetNetMap(nm)
require.Equal(t, nm, s.NetMap())
}

Some files were not shown because too many files have changed in this diff Show more