From 441f1d3bf570eb04000de0509d58380777e66600 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 4 Sep 2019 12:12:02 +0300 Subject: [PATCH] _pkg.dev: drop chain package Duplicated by blockchain code in core. Refs. #307. --- _pkg.dev/chain/chain.go | 137 ------------ _pkg.dev/chain/chaindb.go | 372 --------------------------------- _pkg.dev/chain/chaindb_test.go | 201 ------------------ _pkg.dev/chain/errors.go | 19 -- 4 files changed, 729 deletions(-) delete mode 100644 _pkg.dev/chain/chain.go delete mode 100644 _pkg.dev/chain/chaindb.go delete mode 100644 _pkg.dev/chain/chaindb_test.go delete mode 100644 _pkg.dev/chain/errors.go diff --git a/_pkg.dev/chain/chain.go b/_pkg.dev/chain/chain.go deleted file mode 100644 index f49aedbac..000000000 --- a/_pkg.dev/chain/chain.go +++ /dev/null @@ -1,137 +0,0 @@ -package chain - -import ( - "fmt" - - "github.com/pkg/errors" - - "github.com/CityOfZion/neo-go/pkg/chaincfg" - "github.com/CityOfZion/neo-go/pkg/wire/payload/transaction" - "github.com/CityOfZion/neo-go/pkg/wire/protocol" - - "github.com/CityOfZion/neo-go/pkg/database" - "github.com/CityOfZion/neo-go/pkg/wire/payload" -) - -var ( - // ErrBlockAlreadyExists happens when you try to save the same block twice - ErrBlockAlreadyExists = errors.New("this block has already been saved in the database") - - // ErrFutureBlock happens when you try to save a block that is not the next block sequentially - ErrFutureBlock = errors.New("this is not the next block sequentially, that should be added to the chain") -) - -// Chain represents a blockchain instance -type Chain struct { - Db *Chaindb - height uint32 -} - -// New returns a new chain instance -func New(db database.Database, magic protocol.Magic) (*Chain, error) { - - chain := &Chain{ - Db: &Chaindb{db}, - } - - // Get last header saved to see if this is a fresh database - _, err := chain.Db.GetLastHeader() - if err == nil { - return chain, nil - } - - if err != database.ErrNotFound { - return nil, err - } - - // We have a database.ErrNotFound. Insert the genesisBlock - fmt.Printf("Starting a fresh database for %s\n", magic.String()) - - params, err := chaincfg.NetParams(magic) - if err != nil { - return nil, err - } - err = chain.Db.saveHeader(¶ms.GenesisBlock.BlockBase) - if err != nil { - return nil, err - } - err = chain.Db.saveBlock(params.GenesisBlock, true) - if err != nil { - return nil, err - } - return chain, nil -} - -// ProcessBlock verifies and saves the block in the database -// XXX: for now we will just save without verifying the block -// This function is called by the server and if an error is returned then -// the server informs the sync manager to redownload the block -// XXX:We should also check if the header is already saved in the database -// If not, then we need to validate the header with the rest of the chain -// For now we re-save the header -func (c *Chain) ProcessBlock(block payload.Block) error { - - // Check if we already have this block saved - // XXX: We can optimise by implementing a Has() method - // caching the last block in memory - lastBlock, err := c.Db.GetLastBlock() - if err != nil { - return err - } - if lastBlock.Index > block.Index { - return ErrBlockAlreadyExists - } - - if block.Index > lastBlock.Index+1 { - return ErrFutureBlock - } - - err = c.verifyBlock(block) - if err != nil { - return ValidationError{err.Error()} - } - err = c.Db.saveBlock(block, false) - if err != nil { - return DatabaseError{err.Error()} - } - return nil -} - -// VerifyBlock verifies whether a block is valid according -// to the rules of consensus -func (c *Chain) verifyBlock(block payload.Block) error { - return nil -} - -// VerifyTx verifies whether a transaction is valid according -// to the rules of consensus -func (c *Chain) VerifyTx(tx transaction.Transactioner) error { - return nil -} - -// ProcessHeaders will save the set of headers without validating -func (c *Chain) ProcessHeaders(hdrs []*payload.BlockBase) error { - - err := c.verifyHeaders(hdrs) - if err != nil { - return ValidationError{err.Error()} - } - err = c.Db.saveHeaders(hdrs) - if err != nil { - return DatabaseError{err.Error()} - } - return nil -} - -// verifyHeaders will be used to verify a batch of headers -// should only ever be called during the initial block download -// or when the node receives a HeadersMessage -func (c *Chain) verifyHeaders(hdrs []*payload.BlockBase) error { - return nil -} - -// CurrentHeight returns the index of the block -// at the tip of the chain -func (c Chain) CurrentHeight() uint32 { - return c.height -} diff --git a/_pkg.dev/chain/chaindb.go b/_pkg.dev/chain/chaindb.go deleted file mode 100644 index 5f1a7124e..000000000 --- a/_pkg.dev/chain/chaindb.go +++ /dev/null @@ -1,372 +0,0 @@ -package chain - -import ( - "bufio" - "bytes" - "encoding/binary" - - "github.com/CityOfZion/neo-go/pkg/database" - "github.com/CityOfZion/neo-go/pkg/wire/payload" - "github.com/CityOfZion/neo-go/pkg/wire/payload/transaction" - "github.com/CityOfZion/neo-go/pkg/wire/util" -) - -var ( - // TX is the prefix used when inserting a tx into the db - TX = []byte("TX") - // HEADER is the prefix used when inserting a header into the db - HEADER = []byte("HE") - // LATESTHEADER is the prefix used when inserting the latests header into the db - LATESTHEADER = []byte("LH") - // UTXO is the prefix used when inserting a utxo into the db - UTXO = []byte("UT") - // LATESTBLOCK is the prefix used when inserting the latest block into the db - LATESTBLOCK = []byte("LB") - // BLOCKHASHTX is the prefix used when linking a blockhash to a given tx - BLOCKHASHTX = []byte("BT") - // BLOCKHASHHEIGHT is the prefix used when linking a blockhash to it's height - // This is linked both ways - BLOCKHASHHEIGHT = []byte("BH") - // SCRIPTHASHUTXO is the prefix used when linking a utxo to a scripthash - // This is linked both ways - SCRIPTHASHUTXO = []byte("SU") -) - -// Chaindb is a wrapper around the db interface which adds an extra block chain specific layer on top. -type Chaindb struct { - db database.Database -} - -// This should not be exported for other callers. -// It is safe-guarded by the chain's verification logic -func (c *Chaindb) saveBlock(blk payload.Block, genesis bool) error { - - latestBlockTable := database.NewTable(c.db, LATESTBLOCK) - hashHeightTable := database.NewTable(c.db, BLOCKHASHHEIGHT) - - // Save Txs and link to block hash - err := c.saveTXs(blk.Txs, blk.Hash.Bytes(), genesis) - if err != nil { - return err - } - - // LINK block height to hash - Both ways - // This allows us to fetch a block using it's hash or it's height - // Given the height, we will search the table to get the hash - // We can then fetch all transactions in the tx table, which match that block hash - height := uint32ToBytes(blk.Index) - err = hashHeightTable.Put(height, blk.Hash.Bytes()) - if err != nil { - return err - } - - err = hashHeightTable.Put(blk.Hash.Bytes(), height) - if err != nil { - return err - } - - // Add block as latest block - // This also acts a Commit() for the block. - // If an error occured, then this will be set to the previous block - // This is useful because if the node suddently shut down while saving and the database was not corrupted - // Then the node will see the latestBlock as the last saved block, and re-download the faulty block - // Note: We check for the latest block on startup - return latestBlockTable.Put([]byte(""), blk.Hash.Bytes()) -} - -// Saves a tx and links each tx to the block it was found in -// This should never be exported. Only way to add a tx, is through it's block -func (c *Chaindb) saveTXs(txs []transaction.Transactioner, blockHash []byte, genesis bool) error { - - for txIndex, tx := range txs { - err := c.saveTx(tx, uint32(txIndex), blockHash, genesis) - if err != nil { - return err - } - } - return nil -} - -func (c *Chaindb) saveTx(tx transaction.Transactioner, txIndex uint32, blockHash []byte, genesis bool) error { - - txTable := database.NewTable(c.db, TX) - blockTxTable := database.NewTable(c.db, BLOCKHASHTX) - - // Save the whole tx using it's hash a key - // In order to find a tx in this table, we need to know it's hash - txHash, err := tx.ID() - if err != nil { - return err - } - err = txTable.Put(txHash.Bytes(), tx.BaseTx().Bytes()) - if err != nil { - return err - } - - // LINK TXhash to block - // This allows us to fetch a tx by just knowing what block it was in - // This is useful for when we want to re-construct a block from it's hash - // In order to ge the tx, we must do a prefix search on blockHash - // This will return a set of txHashes. - //We can then use these hashes to search the txtable for the tx's we need - key := bytesConcat(blockHash, uint32ToBytes(txIndex)) - err = blockTxTable.Put(key, txHash.Bytes()) - if err != nil { - return err - } - - // Save all of the utxos in a transaction - // We do this additional save so that we can form a utxo database - // and know when a transaction is a double spend. - utxos := tx.BaseTx().Outputs - for utxoIndex, utxo := range utxos { - err := c.saveUTXO(utxo, uint16(utxoIndex), txHash.Bytes(), blockHash) - if err != nil { - return err - } - } - - // Do not check for spent utxos on the genesis block - if genesis { - return nil - } - - // Remove all spent utxos - // We do this so that once an output has been spent - // It will be removed from the utxo database and cannot be spent again - // If the output was never in the utxo database, this function will return an error - txos := tx.BaseTx().Inputs - for _, txo := range txos { - err := c.removeUTXO(txo) - if err != nil { - return err - } - } - return nil -} - -// saveUTxo will save a utxo and link it to it's transaction and block -func (c *Chaindb) saveUTXO(utxo *transaction.Output, utxoIndex uint16, txHash, blockHash []byte) error { - - utxoTable := database.NewTable(c.db, UTXO) - scripthashUTXOTable := database.NewTable(c.db, SCRIPTHASHUTXO) - - // This is quite messy, we should (if possible) find a way to pass a Writer and Reader interface - // Encode utxo into a buffer - buf := new(bytes.Buffer) - bw := &util.BinWriter{W: buf} - if utxo.Encode(bw); bw.Err != nil { - return bw.Err - } - - // Save UTXO - // In order to find a utxo in the utxoTable - // One must know the txHash that the utxo was in - key := bytesConcat(txHash, uint16ToBytes(utxoIndex)) - if err := utxoTable.Put(key, buf.Bytes()); err != nil { - return err - } - - // LINK utxo to scripthash - // This allows us to find a utxo with the scriptHash - // Since the key starts with scriptHash, we can look for the scriptHash prefix - // and find all utxos for a given scriptHash. - // Additionally, we can search for all utxos for a certain user in a certain block with scriptHash+blockHash - // But this may not be of use to us. However, note that we cannot have just the scriptHash with the utxoIndex - // as this may not be unique. If Kim/Dautt agree, we can change blockHash to blockHeight, which allows us - // To get all utxos above a certain blockHeight. Question is; Would this be useful? - newKey := bytesConcat(utxo.ScriptHash.Bytes(), blockHash, uint16ToBytes(utxoIndex)) - if err := scripthashUTXOTable.Put(newKey, key); err != nil { - return err - } - if err := scripthashUTXOTable.Put(key, newKey); err != nil { - return err - } - return nil -} - -// Remove -func (c *Chaindb) removeUTXO(txo *transaction.Input) error { - - utxoTable := database.NewTable(c.db, UTXO) - scripthashUTXOTable := database.NewTable(c.db, SCRIPTHASHUTXO) - - // Remove spent utxos from utxo database - key := bytesConcat(txo.PrevHash.Bytes(), uint16ToBytes(txo.PrevIndex)) - err := utxoTable.Delete(key) - if err != nil { - return err - } - - // Remove utxos from scripthash table - otherKey, err := scripthashUTXOTable.Get(key) - if err != nil { - return err - } - if err := scripthashUTXOTable.Delete(otherKey); err != nil { - return err - } - if err := scripthashUTXOTable.Delete(key); err != nil { - return err - } - - return nil -} - -// saveHeaders will save a set of headers into the database -func (c *Chaindb) saveHeaders(headers []*payload.BlockBase) error { - - for _, hdr := range headers { - err := c.saveHeader(hdr) - if err != nil { - return err - } - } - return nil -} - -// saveHeader saves a header into the database and updates the latest header -// The headers are saved with their `blockheights` as Key -// If we want to search for a header, we need to know it's index -// Alternatively, we can search the hashHeightTable with the block index to get the hash -// If the block has been saved. -// The reason why headers are saved with their index as Key, is so that we can -// increment the key to find out what block we should fetch next during the initial -// block download, when we are saving thousands of headers -func (c *Chaindb) saveHeader(hdr *payload.BlockBase) error { - - headerTable := database.NewTable(c.db, HEADER) - latestHeaderTable := database.NewTable(c.db, LATESTHEADER) - - index := uint32ToBytes(hdr.Index) - - byt, err := hdr.Bytes() - if err != nil { - return err - } - - err = headerTable.Put(index, byt) - if err != nil { - return err - } - - // Update latest header - return latestHeaderTable.Put([]byte(""), index) -} - -// GetHeaderFromHeight will get a header given it's block height -func (c *Chaindb) GetHeaderFromHeight(index []byte) (*payload.BlockBase, error) { - headerTable := database.NewTable(c.db, HEADER) - hdrBytes, err := headerTable.Get(index) - if err != nil { - return nil, err - } - reader := bytes.NewReader(hdrBytes) - - blockBase := &payload.BlockBase{} - err = blockBase.Decode(reader) - if err != nil { - return nil, err - } - return blockBase, nil -} - -// GetLastHeader will get the header which was saved last in the database -func (c *Chaindb) GetLastHeader() (*payload.BlockBase, error) { - - latestHeaderTable := database.NewTable(c.db, LATESTHEADER) - index, err := latestHeaderTable.Get([]byte("")) - if err != nil { - return nil, err - } - return c.GetHeaderFromHeight(index) -} - -// GetBlockFromHash will return a block given it's hash -func (c *Chaindb) GetBlockFromHash(blockHash []byte) (*payload.Block, error) { - - blockTxTable := database.NewTable(c.db, BLOCKHASHTX) - - // To get a block we need to fetch: - // The transactions (1) - // The header (2) - - // Reconstruct block by fetching it's txs (1) - var txs []transaction.Transactioner - - // Get all Txhashes for this block - txHashes, err := blockTxTable.Prefix(blockHash) - if err != nil { - return nil, err - } - - // Get all Tx's given their hash - txTable := database.NewTable(c.db, TX) - for _, txHash := range txHashes { - - // Fetch tx by it's hash - txBytes, err := txTable.Get(txHash) - if err != nil { - return nil, err - } - reader := bufio.NewReader(bytes.NewReader(txBytes)) - - tx, err := transaction.FromReader(reader) - if err != nil { - return nil, err - } - txs = append(txs, tx) - } - - // Now fetch the header (2) - // We have the block hash, but headers are stored with their `Height` as key. - // We first search the `BlockHashHeight` table to get the height. - //Then we search the headers table with the height - hashHeightTable := database.NewTable(c.db, BLOCKHASHHEIGHT) - height, err := hashHeightTable.Get(blockHash) - if err != nil { - return nil, err - } - hdr, err := c.GetHeaderFromHeight(height) - if err != nil { - return nil, err - } - - // Construct block - block := &payload.Block{ - BlockBase: *hdr, - Txs: txs, - } - return block, nil -} - -// GetLastBlock will return the last block that has been saved -func (c *Chaindb) GetLastBlock() (*payload.Block, error) { - - latestBlockTable := database.NewTable(c.db, LATESTBLOCK) - blockHash, err := latestBlockTable.Get([]byte("")) - if err != nil { - return nil, err - } - return c.GetBlockFromHash(blockHash) -} - -func uint16ToBytes(x uint16) []byte { - index := make([]byte, 2) - binary.BigEndian.PutUint16(index, x) - return index -} - -func uint32ToBytes(x uint32) []byte { - index := make([]byte, 4) - binary.BigEndian.PutUint32(index, x) - return index -} - -func bytesConcat(args ...[]byte) []byte { - var res []byte - for _, arg := range args { - res = append(res, arg...) - } - return res -} diff --git a/_pkg.dev/chain/chaindb_test.go b/_pkg.dev/chain/chaindb_test.go deleted file mode 100644 index b5656d30b..000000000 --- a/_pkg.dev/chain/chaindb_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package chain - -import ( - "bytes" - "math/rand" - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/CityOfZion/neo-go/pkg/database" - "github.com/CityOfZion/neo-go/pkg/wire/payload" - "github.com/CityOfZion/neo-go/pkg/wire/payload/transaction" - "github.com/CityOfZion/neo-go/pkg/wire/util" -) - -var s = rand.NewSource(time.Now().UnixNano()) -var r = rand.New(s) - -func TestLastHeader(t *testing.T) { - _, cdb, hdrs := saveRandomHeaders(t) - - // Select last header from list of headers - lastHeader := hdrs[len(hdrs)-1] - // GetLastHeader from the database - hdr, err := cdb.GetLastHeader() - assert.Nil(t, err) - assert.Equal(t, hdr.Index, lastHeader.Index) - - // Clean up - os.RemoveAll(database.DbDir) -} - -func TestSaveHeader(t *testing.T) { - // save headers then fetch a random element - - db, _, hdrs := saveRandomHeaders(t) - - headerTable := database.NewTable(db, HEADER) - // check that each header was saved - for _, hdr := range hdrs { - index := uint32ToBytes(hdr.Index) - ok, err := headerTable.Has(index) - assert.Nil(t, err) - assert.True(t, ok) - } - - // Clean up - os.RemoveAll(database.DbDir) -} - -func TestSaveBlock(t *testing.T) { - - // Init databases - db, err := database.New("temp.test") - assert.Nil(t, err) - - cdb := &Chaindb{db} - - // Construct block0 and block1 - block0, block1 := twoBlocksLinked(t) - - // Save genesis header - err = cdb.saveHeader(&block0.BlockBase) - assert.Nil(t, err) - - // Save genesis block - err = cdb.saveBlock(block0, true) - assert.Nil(t, err) - - // Test genesis block saved - testBlockWasSaved(t, cdb, block0) - - // Save block1 header - err = cdb.saveHeader(&block1.BlockBase) - assert.Nil(t, err) - - // Save block1 - err = cdb.saveBlock(block1, false) - assert.Nil(t, err) - - // Test block1 was saved - testBlockWasSaved(t, cdb, block1) - - // Clean up - os.RemoveAll(database.DbDir) -} - -func testBlockWasSaved(t *testing.T, cdb *Chaindb, block payload.Block) { - // Fetch last block from database - lastBlock, err := cdb.GetLastBlock() - assert.Nil(t, err) - - // Get byte representation of last block from database - byts, err := lastBlock.Bytes() - assert.Nil(t, err) - - // Get byte representation of block that we saved - blockBytes, err := block.Bytes() - assert.Nil(t, err) - - // Should be equal - assert.True(t, bytes.Equal(byts, blockBytes)) -} - -func randomHeaders(t *testing.T) []*payload.BlockBase { - assert := assert.New(t) - hdrsMsg, err := payload.NewHeadersMessage() - assert.Nil(err) - - for i := 0; i < 2000; i++ { - err = hdrsMsg.AddHeader(randomBlockBase(t)) - assert.Nil(err) - } - - return hdrsMsg.Headers -} - -func randomBlockBase(t *testing.T) *payload.BlockBase { - - base := &payload.BlockBase{ - Version: r.Uint32(), - PrevHash: randUint256(t), - MerkleRoot: randUint256(t), - Timestamp: r.Uint32(), - Index: r.Uint32(), - ConsensusData: r.Uint64(), - NextConsensus: randUint160(t), - Witness: transaction.Witness{ - InvocationScript: []byte{0, 1, 2, 34, 56}, - VerificationScript: []byte{0, 12, 3, 45, 66}, - }, - Hash: randUint256(t), - } - return base -} - -func randomTxs(t *testing.T) []transaction.Transactioner { - - var txs []transaction.Transactioner - for i := 0; i < 10; i++ { - tx := transaction.NewContract(0) - tx.AddInput(transaction.NewInput(randUint256(t), uint16(r.Int()))) - tx.AddOutput(transaction.NewOutput(randUint256(t), r.Int63(), randUint160(t))) - txs = append(txs, tx) - } - return txs -} - -func saveRandomHeaders(t *testing.T) (database.Database, *Chaindb, []*payload.BlockBase) { - db, err := database.New("temp.test") - assert.Nil(t, err) - - cdb := &Chaindb{db} - - hdrs := randomHeaders(t) - - err = cdb.saveHeaders(hdrs) - assert.Nil(t, err) - return db, cdb, hdrs -} - -func randUint256(t *testing.T) util.Uint256 { - slice := make([]byte, 32) - _, err := r.Read(slice) - u, err := util.Uint256DecodeBytes(slice) - assert.Nil(t, err) - return u -} -func randUint160(t *testing.T) util.Uint160 { - slice := make([]byte, 20) - _, err := r.Read(slice) - u, err := util.Uint160DecodeBytes(slice) - assert.Nil(t, err) - return u -} - -// twoBlocksLinked will return two blocks, the second block spends from the utxos in the first -func twoBlocksLinked(t *testing.T) (payload.Block, payload.Block) { - genesisBase := randomBlockBase(t) - genesisTxs := randomTxs(t) - genesisBlock := payload.Block{BlockBase: *genesisBase, Txs: genesisTxs} - - var txs []transaction.Transactioner - - // Form transactions that spend from the genesis block - for _, tx := range genesisTxs { - txHash, err := tx.ID() - assert.Nil(t, err) - newTx := transaction.NewContract(0) - newTx.AddInput(transaction.NewInput(txHash, 0)) - newTx.AddOutput(transaction.NewOutput(randUint256(t), r.Int63(), randUint160(t))) - txs = append(txs, newTx) - } - - nextBase := randomBlockBase(t) - nextBlock := payload.Block{BlockBase: *nextBase, Txs: txs} - - return genesisBlock, nextBlock -} diff --git a/_pkg.dev/chain/errors.go b/_pkg.dev/chain/errors.go deleted file mode 100644 index 627a304ba..000000000 --- a/_pkg.dev/chain/errors.go +++ /dev/null @@ -1,19 +0,0 @@ -package chain - -// ValidationError occurs when verificatio of the object fails -type ValidationError struct { - msg string -} - -func (v ValidationError) Error() string { - return v.msg -} - -// DatabaseError occurs when the chain fails to save the object in the database -type DatabaseError struct { - msg string -} - -func (d DatabaseError) Error() string { - return d.msg -}