Merge pull request #70 from smallstep/max/badger

Update nosql integration

Gopkg.lock

@@ -1,6 +1,14 @@
 # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
 
 
+[[projects]]
+  branch = "master"
+  digest = "1:6716c9fe6333591128e72848f246fc01dc72240e1e64185d8b4e124e7280b35d"
+  name = "github.com/AndreasBriese/bbloom"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "e2d15f34fcf99d5dbb871c820ec73f710fca9815"
+
 [[projects]]
   digest = "1:304cb78c285eaf02ab529ad02a257cad9b4845022915e6c82f87860ac53222d8"
   name = "github.com/alecthomas/gometalinter"
@@ -35,6 +43,29 @@
   revision = "b90dc15cfd220ecf8bbc9043ecb928cef381f011"
   version = "v0.3.4"
 
+[[projects]]
+  digest = "1:21ac9938fb1098b3a7b0dd909fb30878d33231177fac11a2821114eb9c1088ff"
+  name = "github.com/dgraph-io/badger"
+  packages = [
+    ".",
+    "options",
+    "protos",
+    "skl",
+    "table",
+    "y",
+  ]
+  pruneopts = "UT"
+  revision = "391b6d3b93e6014fe8c2971fcc0c1266e47dbbd9"
+  version = "v1.5.3"
+
+[[projects]]
+  branch = "master"
+  digest = "1:6e8109ce247a59ab1eeb5330166c12735f6590de99c9647b6162d11518d32c9a"
+  name = "github.com/dgryski/go-farm"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "6a90982ecee230ff6cba02d5bd386acc030be9d3"
+
 [[projects]]
   digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda"
   name = "github.com/ghodss/yaml"
@@ -51,6 +82,14 @@
   pruneopts = "UT"
   revision = "0ebf7795c516423a110473652e9ba3a59a504863"
 
+[[projects]]
+  digest = "1:ec6f9bf5e274c833c911923c9193867f3f18788c461f76f05f62bb1510e0ae65"
+  name = "github.com/go-sql-driver/mysql"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "72cd26f257d44c1114970e19afddcd812016007e"
+  version = "v1.4.1"
+
 [[projects]]
   digest = "1:b402bb9a24d108a9405a6f34675091b036c8b056aac843bf6ef2389a65c5cf48"
   name = "github.com/gogo/protobuf"
@@ -174,6 +213,28 @@
   revision = "6ca4dbf54d38eea1a992b3c722a76a5d1c4cb25c"
   version = "v0.0.4"
 
+[[projects]]
+  branch = "master"
+  digest = "1:ae08d850ba158ea3ba4a7bb90f8372608172d8920644e5a6693b940a1f4e5d01"
+  name = "github.com/mmcloughlin/avo"
+  packages = [
+    "attr",
+    "build",
+    "buildtags",
+    "gotypes",
+    "internal/prnt",
+    "internal/stack",
+    "ir",
+    "operand",
+    "pass",
+    "printer",
+    "reg",
+    "src",
+    "x86",
+  ]
+  pruneopts = "UT"
+  revision = "2e7d06bc7ada2979f17ccf8ebf486dba23b84fc7"
+
 [[projects]]
   digest = "1:33422d238f147d247752996a26574ac48dcf472976eda7f5134015f06bf16563"
   name = "github.com/modern-go/concurrent"
@@ -302,11 +363,17 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:fd8d9eb07509d8ef47fc82c99646f0b2203b2ba3c240ba77d8c457bb6109836d"
+  digest = "1:f1f1df1e19d55a1ef1f0a63633191e7d2a99993c0a17f945c0b9ebd16b17871b"
   name = "github.com/smallstep/nosql"
-  packages = ["."]
+  packages = [
+    ".",
+    "badger",
+    "bolt",
+    "database",
+    "mysql",
+  ]
   pruneopts = "UT"
-  revision = "d8f68d14f9ae04e0991dce06b44768f2d38dccf8"
+  revision = "5a355c598075a346d9ca9b50ec10e3f86ac66148"
 
 [[projects]]
   branch = "master"
@@ -402,17 +469,30 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:384310e8a567edf6d5406d93318f9460c2d8db1a07ff5b6fece95b224343e7f1"
+  digest = "1:6b4a1c844969280f4d3e36ef4b0762e3522e701c015f688b68ef91c2ea6b5ac7"
   name = "golang.org/x/tools"
   packages = [
     "go/ast/astutil",
     "go/gcexportdata",
+    "go/internal/cgo",
     "go/internal/gcimporter",
+    "go/packages",
     "go/types/typeutil",
+    "internal/fastwalk",
+    "internal/gopathwalk",
+    "internal/semver",
   ]
   pruneopts = "UT"
   revision = "3a10b9bf0a52df7e992a8c3eb712a86d3c896c75"
 
+[[projects]]
+  digest = "1:c25289f43ac4a68d88b02245742347c94f1e108c534dda442188015ff80669b3"
+  name = "google.golang.org/appengine"
+  packages = ["cloudsql"]
+  pruneopts = "UT"
+  revision = "54a98f90d1c46b7731eb8fb305d2a321c30ef610"
+  version = "v1.5.0"
+
 [[projects]]
   branch = "master"
   digest = "1:077c1c599507b3b3e9156d17d36e1e61928ee9b53a5b420f10f28ebd4a0b275c"
@@ -591,6 +671,7 @@
     "github.com/smallstep/cli/token/provision",
     "github.com/smallstep/cli/usage",
     "github.com/smallstep/nosql",
+    "github.com/smallstep/nosql/database",
     "github.com/tsenart/deadcode",
     "github.com/urfave/cli",
     "golang.org/x/crypto/ocsp",

ca/ca.go

@@ -3,6 +3,7 @@ package ca
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"log"
 	"net/http"
 
 	"github.com/go-chi/chi"
@@ -123,7 +124,7 @@ func (ca *CA) Run() error {
 func (ca *CA) Stop() error {
 	ca.renewer.Stop()
 	if err := ca.auth.Shutdown(); err != nil {
-		return err
+		log.Printf("error stopping ca.Authority: %+v\n", err)
 	}
 	return ca.srv.Shutdown()
 }
@@ -131,21 +132,64 @@ func (ca *CA) Stop() error {
 // Reload reloads the configuration of the CA and calls to the server Reload
 // method.
 func (ca *CA) Reload() error {
+	var hasDB bool
+	if ca.config.DB != nil {
+		hasDB = true
+	}
 	if ca.opts.configFile == "" {
 		return errors.New("error reloading ca: configuration file is not set")
 	}
 
 	config, err := authority.LoadConfiguration(ca.opts.configFile)
 	if err != nil {
-		return errors.Wrap(err, "error reloading ca")
+		return errors.Wrap(err, "error reloading ca configuration")
 	}
 
+	logShutDown := func(ss ...string) {
+		for _, s := range ss {
+			log.Println(s)
+		}
+		log.Println("Continuing to serve requests may result in inconsistent state. Shutting Down ...")
+	}
+	logContinue := func(reason string) {
+		log.Println(reason)
+		log.Println("Continuing to run with the original configuration.")
+		log.Println("You can force a restart by sending a SIGTERM signal and then restarting the step-ca.")
+	}
+
+	// Shut down the old authority (shut down the database). If New or Reload
+	// fails then the CA will continue to run but the database will have been
+	// shutdown, which will cause errors.
+	if err := ca.auth.Shutdown(); err != nil {
+		if hasDB {
+			logShutDown("Attempt to shut down the ca.Authority has failed.")
+			return ca.Stop()
+		}
+		logContinue("Reload failed because the ca.Authority could not be shut down.")
+		return err
+	}
 	newCA, err := New(config, WithPassword(ca.opts.password), WithConfigFile(ca.opts.configFile))
 	if err != nil {
+		if hasDB {
+			logShutDown("Attempt to initialize a CA with the new configuration has failed.",
+				"The database has already been shutdown.")
+			return ca.Stop()
+		}
+		logContinue("Reload failed because the CA with new configuration could " +
+			"not be initialized.")
 		return errors.Wrap(err, "error reloading ca")
 	}
 
-	return ca.srv.Reload(newCA.srv)
+	if err = ca.srv.Reload(newCA.srv); err != nil {
+		if hasDB {
+			logShutDown("Attempt to replace the old CA server has failed.",
+				"The database has already been shutdown.")
+			return ca.Stop()
+		}
+		logContinue("Reload failed because server could not be replaced.")
+		return errors.Wrap(err, "error reloading server")
+	}
+	return nil
 }
 
 // getTLSConfig returns a TLSConfig for the CA server with a self-renewing

db/db.go

@@ -3,7 +3,6 @@ package db
 import (
 	"crypto/x509"
 	"encoding/json"
-	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -22,7 +21,9 @@ var ErrAlreadyExists = errors.New("already exists")
 // Config represents the JSON attributes used for configuring a step-ca DB.
 type Config struct {
 	Type       string `json:"type"`
-	Path string `json:"path"`
+	DataSource string `json:"dataSource"`
+	ValueDir   string `json:"valueDir"`
+	Database   string `json:"database"`
 }
 
 // AuthDB is an interface over an Authority DB client that implements a nosql.DB interface.
@@ -36,6 +37,7 @@ type AuthDB interface {
 // DB is a wrapper over the nosql.DB interface.
 type DB struct {
 	nosql.DB
+	isUp bool
 }
 
 // New returns a new database client that implements the AuthDB interface.
@@ -44,15 +46,10 @@ func New(c *Config) (AuthDB, error) {
 		return new(NoopDB), nil
 	}
 
-	var db nosql.DB
+	db, err := nosql.New(c.Type, c.DataSource, nosql.WithDatabase(c.Database),
-	switch strings.ToLower(c.Type) {
+		nosql.WithValueDir(c.ValueDir))
-	case "bbolt":
+	if err != nil {
-		db = &nosql.BoltDB{}
+		return nil, errors.Wrapf(err, "Error opening database of Type %s with source %s", c.Type, c.DataSource)
-		if err := db.Open(c.Path); err != nil {
-			return nil, err
-		}
-	default:
-		return nil, errors.Errorf("unsupported db.type '%s'", c.Type)
 	}
 
 	tables := [][]byte{revokedCertsTable, certsTable}
@@ -63,7 +60,7 @@ func New(c *Config) (AuthDB, error) {
 		}
 	}
 
-	return &DB{db}, nil
+	return &DB{db, true}, nil
 }
 
 // RevokedCertificateInfo contains information regarding the certificate
@@ -131,8 +128,11 @@ func (db *DB) StoreCertificate(crt *x509.Certificate) error {
 
 // Shutdown sends a shutdown message to the database.
 func (db *DB) Shutdown() error {
+	if db.isUp {
 		if err := db.Close(); err != nil {
 			return errors.Wrap(err, "database shutdown error")
 		}
+		db.isUp = false
+	}
 	return nil
 }

db/db_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/smallstep/assert"
-	"github.com/smallstep/nosql"
+	"github.com/smallstep/nosql/database"
 )
 
 type MockNoSQLDB struct {
@@ -13,13 +13,13 @@ type MockNoSQLDB struct {
 	ret1, ret2  interface{}
 	get         func(bucket, key []byte) ([]byte, error)
 	set         func(bucket, key, value []byte) error
-	open        func(path string) error
+	open        func(dataSourceName string, opt ...database.Option) error
 	close       func() error
 	createTable func(bucket []byte) error
 	deleteTable func(bucket []byte) error
 	del         func(bucket, key []byte) error
-	list        func(bucket []byte) ([]*nosql.Entry, error)
+	list        func(bucket []byte) ([]*database.Entry, error)
-	update      func(tx *nosql.Tx) error
+	update      func(tx *database.Tx) error
 }
 
 func (m *MockNoSQLDB) Get(bucket, key []byte) ([]byte, error) {
@@ -39,9 +39,9 @@ func (m *MockNoSQLDB) Set(bucket, key, value []byte) error {
 	return m.err
 }
 
-func (m *MockNoSQLDB) Open(path string) error {
+func (m *MockNoSQLDB) Open(dataSourceName string, opt ...database.Option) error {
 	if m.open != nil {
-		return m.open(path)
+		return m.open(dataSourceName, opt...)
 	}
 	return m.err
 }
@@ -74,14 +74,14 @@ func (m *MockNoSQLDB) Del(bucket, key []byte) error {
 	return m.err
 }
 
-func (m *MockNoSQLDB) List(bucket []byte) ([]*nosql.Entry, error) {
+func (m *MockNoSQLDB) List(bucket []byte) ([]*database.Entry, error) {
 	if m.list != nil {
 		return m.list(bucket)
 	}
-	return m.ret1.([]*nosql.Entry), m.err
+	return m.ret1.([]*database.Entry), m.err
 }
 
-func (m *MockNoSQLDB) Update(tx *nosql.Tx) error {
+func (m *MockNoSQLDB) Update(tx *database.Tx) error {
 	if m.update != nil {
 		return m.update(tx)
 	}
@@ -100,16 +100,16 @@ func TestIsRevoked(t *testing.T) {
 		},
 		"false/ErrNotFound": {
 			key: "sn",
-			db:  &DB{&MockNoSQLDB{err: nosql.ErrNotFound, ret1: nil}},
+			db:  &DB{&MockNoSQLDB{err: database.ErrNotFound, ret1: nil}, true},
 		},
 		"error/checking bucket": {
 			key: "sn",
-			db:  &DB{&MockNoSQLDB{err: errors.New("force"), ret1: nil}},
+			db:  &DB{&MockNoSQLDB{err: errors.New("force"), ret1: nil}, true},
 			err: errors.New("error checking revocation bucket: force"),
 		},
 		"true": {
 			key:       "sn",
-			db:        &DB{&MockNoSQLDB{ret1: []byte("value")}},
+			db:        &DB{&MockNoSQLDB{ret1: []byte("value")}, true},
 			isRevoked: true,
 		},
 	}
@@ -140,7 +140,7 @@ func TestRevoke(t *testing.T) {
 				get: func(bucket []byte, sn []byte) ([]byte, error) {
 					return nil, errors.New("force IsRevoked")
 				},
-			}},
+			}, true},
 			err: errors.New("error checking revocation bucket: force IsRevoked"),
 		},
 		"error/was already revoked": {
@@ -149,31 +149,31 @@ func TestRevoke(t *testing.T) {
 				get: func(bucket []byte, sn []byte) ([]byte, error) {
 					return nil, nil
 				},
-			}},
+			}, true},
 			err: ErrAlreadyExists,
 		},
 		"error/database set": {
 			rci: &RevokedCertificateInfo{Serial: "sn"},
 			db: &DB{&MockNoSQLDB{
 				get: func(bucket []byte, sn []byte) ([]byte, error) {
-					return nil, nosql.ErrNotFound
+					return nil, database.ErrNotFound
 				},
 				set: func(bucket []byte, key []byte, value []byte) error {
 					return errors.New("force")
 				},
-			}},
+			}, true},
 			err: errors.New("database Set error: force"),
 		},
 		"ok": {
 			rci: &RevokedCertificateInfo{Serial: "sn"},
 			db: &DB{&MockNoSQLDB{
 				get: func(bucket []byte, sn []byte) ([]byte, error) {
-					return nil, nosql.ErrNotFound
+					return nil, database.ErrNotFound
 				},
 				set: func(bucket []byte, key []byte, value []byte) error {
 					return nil
 				},
-			}},
+			}, true},
 		},
 	}
 	for name, tc := range tests {