From 1c087539158f5d44b81df4f68c773918b85713f0 Mon Sep 17 00:00:00 2001 From: Vsevolod Brekelov Date: Tue, 5 Nov 2019 15:13:26 +0300 Subject: [PATCH 1/3] gomod: fix --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 6b4ad7c72..668cc6bc4 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,6 @@ require ( github.com/gomodule/redigo v2.0.0+incompatible // indirect github.com/mattn/go-colorable v0.1.2 // indirect github.com/mattn/go-isatty v0.0.9 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/mr-tron/base58 v1.1.2 github.com/nspcc-dev/rfc6979 v0.1.0 github.com/onsi/gomega v1.4.2 // indirect From 11ce73af28cad97bdff36d9d73bd9f9002d59003 Mon Sep 17 00:00:00 2001 From: Vsevolod Brekelov Date: Tue, 5 Nov 2019 15:22:07 +0300 Subject: [PATCH 2/3] server: add log-path and address configuration - LogPath can be configured through config - node,rpc and monitoring address can be configured thought command line or config --- cli/server/server.go | 57 +++++++++++++++++++++--- config/config.go | 2 + config/protocol.mainnet.yml | 4 ++ config/protocol.privnet.docker.four.yml | 4 ++ config/protocol.privnet.docker.one.yml | 4 ++ config/protocol.privnet.docker.three.yml | 4 ++ config/protocol.privnet.docker.two.yml | 4 ++ config/protocol.privnet.yml | 4 ++ config/protocol.testnet.yml | 4 ++ config/protocol.unit_testnet.yml | 4 ++ pkg/network/metrics/metrics.go | 3 +- pkg/network/server.go | 4 +- pkg/network/server_config.go | 10 +++-- pkg/network/server_test.go | 2 +- pkg/rpc/server.go | 2 +- 15 files changed, 99 insertions(+), 13 deletions(-) diff --git a/cli/server/server.go b/cli/server/server.go index 9aacdca66..56a3b1215 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "os/signal" + "path" "github.com/CityOfZion/neo-go/config" "github.com/CityOfZion/neo-go/pkg/core" @@ -107,11 +108,35 @@ func getConfigFromContext(ctx *cli.Context) (config.Config, error) { return config.Load(configPath, net) } -// handleLoggingParams enables debugging output is that's requested by the user. -func handleLoggingParams(ctx *cli.Context) { +// handleLoggingParams reads logging parameters. +// If user selected debug level -- function enables it. +// If logPath is configured -- function creates dir and file for logging. +func handleLoggingParams(ctx *cli.Context, cfg config.ApplicationConfiguration) error { if ctx.Bool("debug") { log.SetLevel(log.DebugLevel) } + + if logPath := cfg.LogPath; logPath != "" { + if err := makeDir(logPath); err != nil { + return err + } + f, err := os.Create(logPath) + if err != nil { + return err + } + log.SetOutput(f) + } + return nil +} + +func makeDir(filePath string) error { + fileName := filePath + dir := path.Dir(fileName) + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + return fmt.Errorf("could not create dir for logger: %v", err) + } + return nil } func getCountAndSkipFromContext(ctx *cli.Context) (uint32, uint32) { @@ -125,7 +150,9 @@ func dumpDB(ctx *cli.Context) error { if err != nil { return cli.NewExitError(err, 1) } - handleLoggingParams(ctx) + if err := handleLoggingParams(ctx, cfg.ApplicationConfiguration); err != nil { + return cli.NewExitError(err, 1) + } count, skip := getCountAndSkipFromContext(ctx) var outStream = os.Stdout @@ -173,7 +200,9 @@ func restoreDB(ctx *cli.Context) error { if err != nil { return err } - handleLoggingParams(ctx) + if err := handleLoggingParams(ctx, cfg.ApplicationConfiguration); err != nil { + return cli.NewExitError(err, 1) + } count, skip := getCountAndSkipFromContext(ctx) var inStream = os.Stdin @@ -234,7 +263,9 @@ func startServer(ctx *cli.Context) error { if err != nil { return err } - handleLoggingParams(ctx) + if err := handleLoggingParams(ctx, cfg.ApplicationConfiguration); err != nil { + return err + } grace, cancel := context.WithCancel(newGraceContext()) defer cancel() @@ -246,6 +277,7 @@ func startServer(ctx *cli.Context) error { return err } + configureAddresses(cfg.ApplicationConfiguration) server := network.NewServer(serverConfig, chain) rpcServer := rpc.NewServer(chain, cfg.ApplicationConfiguration.RPC, server) errChan := make(chan error) @@ -285,6 +317,21 @@ Main: return nil } +// configureAddresses sets up addresses for RPC and Monitoring depending from the provided config. +// In case RPC or Monitoring Address provided each of them will use it. +// In case global Address (of the node) provided and RPC/Monitoring don't have configured addresses they will +// use global one. So Node and RPC and Monitoring will run on one address. +func configureAddresses(cfg config.ApplicationConfiguration) { + if cfg.Address != "" { + if cfg.RPC.Address == "" { + cfg.RPC.Address = cfg.Address + } + if cfg.Monitoring.Address == "" { + cfg.Monitoring.Address = cfg.Address + } + } +} + // initBlockChain initializes BlockChain with preselected DB. func initBlockChain(cfg config.Config) (*core.Blockchain, error) { store, err := storage.NewStore(cfg.ApplicationConfiguration.DBConfiguration) diff --git a/config/config.go b/config/config.go index acaf188b4..adc038951 100644 --- a/config/config.go +++ b/config/config.go @@ -66,7 +66,9 @@ type ( // ApplicationConfiguration config specific to the node. ApplicationConfiguration struct { + LogPath string `yaml:"LogPath"` DBConfiguration storage.DBConfiguration `yaml:"DBConfiguration"` + Address string `yaml:"Address"` NodePort uint16 `yaml:"NodePort"` Relay bool `yaml:"Relay"` DialTimeout time.Duration `yaml:"DialTimeout"` diff --git a/config/protocol.mainnet.yml b/config/protocol.mainnet.yml index ba284513a..579df4d17 100644 --- a/config/protocol.mainnet.yml +++ b/config/protocol.mainnet.yml @@ -31,6 +31,8 @@ ProtocolConfiguration: VerifyTransactions: false ApplicationConfiguration: + # LogPath could be set up in case you need stdout logs to some proper file. + # LogPath: "./log/neogo.log" DBConfiguration: Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. @@ -42,6 +44,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/mainnet.bolt" + # Uncomment in order to set up custom address for node. + # Address: 127.0.0.1 NodePort: 10333 Relay: true DialTimeout: 3 diff --git a/config/protocol.privnet.docker.four.yml b/config/protocol.privnet.docker.four.yml index 40b7e0437..aba719273 100644 --- a/config/protocol.privnet.docker.four.yml +++ b/config/protocol.privnet.docker.four.yml @@ -19,6 +19,8 @@ ProtocolConfiguration: VerifyTransactions: true ApplicationConfiguration: + # LogPath could be set up in case you need stdout logs to some proper file. + # LogPath: "./log/neogo.log" DBConfiguration: Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. @@ -30,6 +32,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" + # Uncomment in order to set up custom address for node. + # Address: 127.0.0.1 NodePort: 20337 Relay: true DialTimeout: 3 diff --git a/config/protocol.privnet.docker.one.yml b/config/protocol.privnet.docker.one.yml index bee278bff..1e2365ce9 100644 --- a/config/protocol.privnet.docker.one.yml +++ b/config/protocol.privnet.docker.one.yml @@ -16,6 +16,8 @@ ProtocolConfiguration: VerifyTransactions: true ApplicationConfiguration: + # LogPath could be set up in case you need stdout logs to some proper file. + # LogPath: "./log/neogo.log" DBConfiguration: Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. @@ -27,6 +29,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" + # Uncomment in order to set up custom address for node. + # Address: 127.0.0.1 NodePort: 20334 Relay: true DialTimeout: 3 diff --git a/config/protocol.privnet.docker.three.yml b/config/protocol.privnet.docker.three.yml index 21c214ea6..9d1c82ef7 100644 --- a/config/protocol.privnet.docker.three.yml +++ b/config/protocol.privnet.docker.three.yml @@ -16,6 +16,8 @@ ProtocolConfiguration: VerifyTransactions: true ApplicationConfiguration: + # LogPath could be set up in case you need stdout logs to some proper file. + # LogPath: "./log/neogo.log" DBConfiguration: Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. @@ -27,6 +29,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" + # Uncomment in order to set up custom address for node. + # Address: 127.0.0.1 NodePort: 20336 Relay: true DialTimeout: 3 diff --git a/config/protocol.privnet.docker.two.yml b/config/protocol.privnet.docker.two.yml index 546f94f27..7f5bf917d 100644 --- a/config/protocol.privnet.docker.two.yml +++ b/config/protocol.privnet.docker.two.yml @@ -16,6 +16,8 @@ ProtocolConfiguration: VerifyTransactions: true ApplicationConfiguration: + # LogPath could be set up in case you need stdout logs to some proper file. + # LogPath: "./log/neogo.log" DBConfiguration: Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. @@ -27,6 +29,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" + # Uncomment in order to set up custom address for node. + # Address: 127.0.0.1 NodePort: 20335 Relay: true DialTimeout: 3 diff --git a/config/protocol.privnet.yml b/config/protocol.privnet.yml index 696739465..992e62aeb 100644 --- a/config/protocol.privnet.yml +++ b/config/protocol.privnet.yml @@ -22,6 +22,8 @@ ProtocolConfiguration: VerifyTransactions: true ApplicationConfiguration: + # LogPath could be set up in case you need stdout logs to some proper file. + # LogPath: "./log/neogo.log" DBConfiguration: Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. @@ -33,6 +35,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/privnet.bolt" + # Uncomment in order to set up custom address for node. + # Address: 127.0.0.1 NodePort: 20332 Relay: true DialTimeout: 3 diff --git a/config/protocol.testnet.yml b/config/protocol.testnet.yml index f3de694ec..47ba9c345 100644 --- a/config/protocol.testnet.yml +++ b/config/protocol.testnet.yml @@ -31,6 +31,8 @@ ProtocolConfiguration: VerifyTransactions: false ApplicationConfiguration: + # LogPath could be set up in case you need stdout logs to some proper file. + # LogPath: "./log/neogo.log" DBConfiguration: Type: "leveldb" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. @@ -42,6 +44,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/testnet.bolt" + # Uncomment in order to set up custom address for node. + # Address: 127.0.0.1 NodePort: 20333 Relay: true DialTimeout: 3 diff --git a/config/protocol.unit_testnet.yml b/config/protocol.unit_testnet.yml index 709c52f26..6a76288b1 100644 --- a/config/protocol.unit_testnet.yml +++ b/config/protocol.unit_testnet.yml @@ -21,6 +21,8 @@ ProtocolConfiguration: VerifyTransactions: true ApplicationConfiguration: + # LogPath could be set up in case you need stdout logs to some proper file. + # LogPath: "./log/neogo.log" DBConfiguration: Type: "inmemory" #other options: 'inmemory','redis','boltdb'. # DB type options. Uncomment those you need in case you want to switch DB type. @@ -32,6 +34,8 @@ ApplicationConfiguration: # DB: 0 # BoltDBOptions: # FilePath: "./chains/unit_testnet.bolt" + # Uncomment in order to set up custom address for node. + # Address: 127.0.0.1 NodePort: 20333 Relay: true DialTimeout: 3 diff --git a/pkg/network/metrics/metrics.go b/pkg/network/metrics/metrics.go index 254bd84ba..90bed7a57 100644 --- a/pkg/network/metrics/metrics.go +++ b/pkg/network/metrics/metrics.go @@ -18,6 +18,7 @@ type Service struct { // Additional information about Prometheus could be found here: https://prometheus.io/docs/guides/go-application. type PrometheusConfig struct { Enabled bool `yaml:"Enabled"` + Address string `yaml:"Address"` Port string `yaml:"Port"` } @@ -25,7 +26,7 @@ type PrometheusConfig struct { func NewMetricsService(cfg PrometheusConfig) *Service { return &Service{ &http.Server{ - Addr: ":" + cfg.Port, + Addr: cfg.Address + ":" + cfg.Port, Handler: promhttp.Handler(), }, cfg, } diff --git a/pkg/network/server.go b/pkg/network/server.go index f66145a6c..be9cb7971 100644 --- a/pkg/network/server.go +++ b/pkg/network/server.go @@ -104,7 +104,7 @@ func NewServer(config ServerConfig, chain core.Blockchainer) *Server { s.AttemptConnPeers = defaultAttemptConnPeers } - s.transport = NewTCPTransport(s, fmt.Sprintf(":%d", config.ListenTCP)) + s.transport = NewTCPTransport(s, fmt.Sprintf("%s:%d", config.Address, config.Port)) s.discovery = NewDefaultDiscovery( s.DialTimeout, s.transport, @@ -282,7 +282,7 @@ func (s *Server) startProtocol(p Peer) { func (s *Server) sendVersion(p Peer) error { payload := payload.NewVersion( s.id, - s.ListenTCP, + s.Port, s.UserAgent, s.chain.BlockHeight(), s.Relay, diff --git a/pkg/network/server_config.go b/pkg/network/server_config.go index b0e4332eb..f101ba27c 100644 --- a/pkg/network/server_config.go +++ b/pkg/network/server_config.go @@ -27,8 +27,11 @@ type ( // The user agent of the server. UserAgent string - // The listen address of the TCP server. - ListenTCP uint16 + // Address. Example: "127.0.0.1". + Address string + + // Port. Example: 20332. + Port uint16 // The network mode the server will operate on. // ModePrivNet docker private network. @@ -62,7 +65,8 @@ func NewServerConfig(cfg config.Config) ServerConfig { return ServerConfig{ UserAgent: cfg.GenerateUserAgent(), - ListenTCP: appConfig.NodePort, + Address: appConfig.Address, + Port: appConfig.NodePort, Net: protoConfig.Magic, Relay: appConfig.Relay, Seeds: protoConfig.SeedList, diff --git a/pkg/network/server_test.go b/pkg/network/server_test.go index 70101d89d..bcde70b8c 100644 --- a/pkg/network/server_test.go +++ b/pkg/network/server_test.go @@ -14,7 +14,7 @@ func TestSendVersion(t *testing.T) { s = newTestServer() p = newLocalPeer(t) ) - s.ListenTCP = 3000 + s.Port = 3000 s.UserAgent = "/test/" p.messageHandler = func(t *testing.T, msg *Message) { diff --git a/pkg/rpc/server.go b/pkg/rpc/server.go index eb7304264..474e9103c 100644 --- a/pkg/rpc/server.go +++ b/pkg/rpc/server.go @@ -179,7 +179,7 @@ Methods: case "getversion": getversionCalled.Inc() results = result.Version{ - Port: s.coreServer.ListenTCP, + Port: s.coreServer.Port, Nonce: s.coreServer.ID(), UserAgent: s.coreServer.UserAgent, } From d799c98cfec93d9adcc4edefbbda062b7b7cfe25 Mon Sep 17 00:00:00 2001 From: Vsevolod Brekelov Date: Wed, 6 Nov 2019 17:12:33 +0300 Subject: [PATCH 3/3] io: move common function and add unit tests for it --- cli/server/server.go | 13 +---------- pkg/core/storage/boltdb_store.go | 8 +++---- pkg/io/fileWriter.go | 18 +++++++++++++++ pkg/io/fileWriter_test.go | 39 ++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 17 deletions(-) create mode 100644 pkg/io/fileWriter.go create mode 100644 pkg/io/fileWriter_test.go diff --git a/cli/server/server.go b/cli/server/server.go index 56a3b1215..141e3220a 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "os/signal" - "path" "github.com/CityOfZion/neo-go/config" "github.com/CityOfZion/neo-go/pkg/core" @@ -117,7 +116,7 @@ func handleLoggingParams(ctx *cli.Context, cfg config.ApplicationConfiguration) } if logPath := cfg.LogPath; logPath != "" { - if err := makeDir(logPath); err != nil { + if err := io.MakeDirForFile(logPath, "logger"); err != nil { return err } f, err := os.Create(logPath) @@ -129,16 +128,6 @@ func handleLoggingParams(ctx *cli.Context, cfg config.ApplicationConfiguration) return nil } -func makeDir(filePath string) error { - fileName := filePath - dir := path.Dir(fileName) - err := os.MkdirAll(dir, os.ModePerm) - if err != nil { - return fmt.Errorf("could not create dir for logger: %v", err) - } - return nil -} - func getCountAndSkipFromContext(ctx *cli.Context) (uint32, uint32) { count := uint32(ctx.Uint("count")) skip := uint32(ctx.Uint("skip")) diff --git a/pkg/core/storage/boltdb_store.go b/pkg/core/storage/boltdb_store.go index 93a8ae6e9..c0c1e5f9e 100644 --- a/pkg/core/storage/boltdb_store.go +++ b/pkg/core/storage/boltdb_store.go @@ -4,8 +4,8 @@ import ( "bytes" "fmt" "os" - "path" + "github.com/CityOfZion/neo-go/pkg/io" "github.com/etcd-io/bbolt" log "github.com/sirupsen/logrus" "github.com/syndtr/goleveldb/leveldb/util" @@ -30,10 +30,8 @@ func NewBoltDBStore(cfg BoltDBOptions) (*BoltDBStore, error) { var opts *bbolt.Options // should be exposed via BoltDBOptions if anything needed fileMode := os.FileMode(0600) // should be exposed via BoltDBOptions if anything needed fileName := cfg.FilePath - dir := path.Dir(fileName) - err := os.MkdirAll(dir, os.ModePerm) - if err != nil { - return nil, fmt.Errorf("could not create dir for BoltDB: %v", err) + if err := io.MakeDirForFile(fileName, "BoltDB"); err != nil { + return nil, err } db, err := bbolt.Open(fileName, fileMode, opts) if err != nil { diff --git a/pkg/io/fileWriter.go b/pkg/io/fileWriter.go new file mode 100644 index 000000000..35d87f34f --- /dev/null +++ b/pkg/io/fileWriter.go @@ -0,0 +1,18 @@ +package io + +import ( + "fmt" + "os" + "path" +) + +// MakeDirForFile creates directory provided in filePath. +func MakeDirForFile(filePath string, creator string) error { + fileName := filePath + dir := path.Dir(fileName) + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + return fmt.Errorf("could not create dir for %s: %v", creator, err) + } + return nil +} diff --git a/pkg/io/fileWriter_test.go b/pkg/io/fileWriter_test.go new file mode 100644 index 000000000..a2c8f7e2d --- /dev/null +++ b/pkg/io/fileWriter_test.go @@ -0,0 +1,39 @@ +package io + +import ( + "io/ioutil" + "os" + "path" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMakeDirForFile_HappyPath(t *testing.T) { + tempDir, err := ioutil.TempDir("", "test") + require.NoError(t, err) + + filePath := tempDir + "/testDir/testFile.test" + err = MakeDirForFile(filePath, "test") + defer removeDir(t, tempDir) + require.NoError(t, err) + + _, errChDir := os.Create(filePath) + require.NoError(t, errChDir) +} + +func removeDir(t *testing.T, dirName string) { + err := os.RemoveAll(dirName) + require.NoError(t, err) +} + +func TestMakeDirForFile_Negative(t *testing.T) { + file, err := ioutil.TempFile("", "test") + require.NoError(t, err) + + filePath := file.Name() + "/error" + dir := path.Dir(filePath) + err = MakeDirForFile(filePath, "test") + defer removeDir(t, dir) + require.Errorf(t, err, "could not create dir for test: mkdir %s : not a directory", filePath) +}