Merge pull request #211 from hermeznetwork/feature/update-general

Feature/update general

db/statedb/statedb.go

@@ -5,10 +5,10 @@ import (
 	"fmt"
 	"math/big"
 	"os"
-	"os/exec"
 	"strconv"
 
 	"github.com/hermeznetwork/hermez-node/common"
+	"github.com/hermeznetwork/hermez-node/log"
 	"github.com/iden3/go-merkletree"
 	"github.com/iden3/go-merkletree/db"
 	"github.com/iden3/go-merkletree/db/pebble"
@@ -115,7 +115,7 @@ func NewStateDB(path string, typ TypeStateDB, nLevels int) (*StateDB, error) {
 	}
 
 	// make reset (get checkpoint) at currentBatch
-	err = sdb.Reset(sdb.currentBatch)
+	err = sdb.reset(sdb.currentBatch, false)
 	if err != nil {
 		return nil, err
 	}
@@ -174,6 +174,8 @@ func (s *StateDB) MakeCheckpoint() error {
 		if err != nil {
 			return err
 		}
+	} else if err != nil && !os.IsNotExist(err) {
+		return err
 	}
 
 	// execute Checkpoint
@@ -196,14 +198,59 @@ func (s *StateDB) DeleteCheckpoint(batchNum common.BatchNum) error {
 	return os.RemoveAll(checkpointPath)
 }
 
+func pebbleMakeCheckpoint(source, dest string) error {
+	// Remove dest folder (if it exists) before doing the checkpoint
+	if _, err := os.Stat(dest); !os.IsNotExist(err) {
+		err := os.RemoveAll(dest)
+		if err != nil {
+			return err
+		}
+	} else if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	sto, err := pebble.NewPebbleStorage(source, false)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		errClose := sto.Pebble().Close()
+		if errClose != nil {
+			log.Errorw("Pebble.Close", "err", errClose)
+		}
+	}()
+
+	// execute Checkpoint
+	err = sto.Pebble().Checkpoint(dest)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // Reset resets the StateDB to the checkpoint at the given batchNum. Reset
 // does not delete the checkpoints between old current and the new current,
 // those checkpoints will remain in the storage, and eventually will be
 // deleted when MakeCheckpoint overwrites them.
 func (s *StateDB) Reset(batchNum common.BatchNum) error {
+	return s.reset(batchNum, true)
+}
+
+// reset resets the StateDB to the checkpoint at the given batchNum. Reset
+// does not delete the checkpoints between old current and the new current,
+// those checkpoints will remain in the storage, and eventually will be
+// deleted when MakeCheckpoint overwrites them.  `closeCurrent` will close the
+// currently opened db before doing the reset.
+func (s *StateDB) reset(batchNum common.BatchNum, closeCurrent bool) error {
 	checkpointPath := s.path + PathBatchNum + strconv.Itoa(int(batchNum))
 	currentPath := s.path + PathCurrent
 
+	if closeCurrent {
+		if err := s.db.Pebble().Close(); err != nil {
+			return err
+		}
+	}
 	// remove 'current'
 	err := os.RemoveAll(currentPath)
 	if err != nil {
@@ -222,8 +269,7 @@ func (s *StateDB) Reset(batchNum common.BatchNum) error {
 	}
 
 	// copy 'BatchNumX' to 'current'
-	cmd := exec.Command("cp", "-r", checkpointPath, currentPath) //nolint:gosec
-	err = cmd.Run()
+	err = pebbleMakeCheckpoint(checkpointPath, currentPath)
 	if err != nil {
 		return err
 	}
@@ -482,20 +528,21 @@ func (l *LocalStateDB) Reset(batchNum common.BatchNum, fromSynchronizer bool) er
 			return fmt.Errorf("Checkpoint not exist in Synchronizer")
 		}
 
+		if err := l.db.Pebble().Close(); err != nil {
+			return err
+		}
 		// remove 'current'
 		err := os.RemoveAll(currentPath)
 		if err != nil {
 			return err
 		}
 		// copy synchronizer'BatchNumX' to 'current'
-		cmd := exec.Command("cp", "-r", synchronizerCheckpointPath, currentPath) //nolint:gosec
-		err = cmd.Run()
+		err = pebbleMakeCheckpoint(synchronizerCheckpointPath, currentPath)
 		if err != nil {
 			return err
 		}
-		// copy synchronizer-'BatchNumX' to 'BatchNumX'
-		cmd = exec.Command("cp", "-r", synchronizerCheckpointPath, checkpointPath) //nolint:gosec
-		err = cmd.Run()
+		// copy synchronizer'BatchNumX' to 'BatchNumX'
+		err = pebbleMakeCheckpoint(synchronizerCheckpointPath, checkpointPath)
 		if err != nil {
 			return err
 		}
@@ -522,5 +569,5 @@ func (l *LocalStateDB) Reset(batchNum common.BatchNum, fromSynchronizer bool) er
 		return nil
 	}
 	// use checkpoint from LocalStateDB
-	return l.StateDB.Reset(batchNum)
+	return l.StateDB.reset(batchNum, true)
 }

db/statedb/statedb_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/big"
+	"os"
 	"testing"
 
 	ethCrypto "github.com/ethereum/go-ethereum/crypto"
@@ -38,6 +39,7 @@ func newAccount(t *testing.T, i int) *common.Account {
 func TestNewStateDBIntermediateState(t *testing.T) {
 	dir, err := ioutil.TempDir("", "tmpdb")
 	require.Nil(t, err)
+	defer assert.Nil(t, os.RemoveAll(dir))
 
 	sdb, err := NewStateDB(dir, TypeTxSelector, 0)
 	assert.Nil(t, err)
@@ -119,6 +121,7 @@ func TestNewStateDBIntermediateState(t *testing.T) {
 func TestStateDBWithoutMT(t *testing.T) {
 	dir, err := ioutil.TempDir("", "tmpdb")
 	require.Nil(t, err)
+	defer assert.Nil(t, os.RemoveAll(dir))
 
 	sdb, err := NewStateDB(dir, TypeTxSelector, 0)
 	assert.Nil(t, err)
@@ -172,6 +175,7 @@ func TestStateDBWithoutMT(t *testing.T) {
 func TestStateDBWithMT(t *testing.T) {
 	dir, err := ioutil.TempDir("", "tmpdb")
 	require.Nil(t, err)
+	defer assert.Nil(t, os.RemoveAll(dir))
 
 	sdb, err := NewStateDB(dir, TypeSynchronizer, 32)
 	assert.Nil(t, err)
@@ -223,6 +227,7 @@ func TestStateDBWithMT(t *testing.T) {
 func TestCheckpoints(t *testing.T) {
 	dir, err := ioutil.TempDir("", "sdb")
 	require.Nil(t, err)
+	defer assert.Nil(t, os.RemoveAll(dir))
 
 	sdb, err := NewStateDB(dir, TypeSynchronizer, 32)
 	assert.Nil(t, err)
@@ -290,6 +295,7 @@ func TestCheckpoints(t *testing.T) {
 	// Create a LocalStateDB from the initial StateDB
 	dirLocal, err := ioutil.TempDir("", "ldb")
 	require.Nil(t, err)
+	defer assert.Nil(t, os.RemoveAll(dirLocal))
 	ldb, err := NewLocalStateDB(dirLocal, sdb, TypeBatchBuilder, 32)
 	assert.Nil(t, err)
 
@@ -310,6 +316,7 @@ func TestCheckpoints(t *testing.T) {
 	// Create a 2nd LocalStateDB from the initial StateDB
 	dirLocal2, err := ioutil.TempDir("", "ldb2")
 	require.Nil(t, err)
+	defer assert.Nil(t, os.RemoveAll(dirLocal2))
 	ldb2, err := NewLocalStateDB(dirLocal2, sdb, TypeBatchBuilder, 32)
 	assert.Nil(t, err)
 

db/statedb/txprocessors.go

@@ -2,14 +2,16 @@ package statedb
 
 import (
 	"errors"
+	"io/ioutil"
 	"math/big"
+	"os"
 
 	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/hermez-node/log"
 	"github.com/iden3/go-iden3-crypto/babyjub"
 	"github.com/iden3/go-merkletree"
 	"github.com/iden3/go-merkletree/db"
-	"github.com/iden3/go-merkletree/db/memory"
+	"github.com/iden3/go-merkletree/db/pebble"
 )
 
 var (
@@ -61,7 +63,20 @@ func (s *StateDB) ProcessTxs(l1usertxs, l1coordinatortxs []common.L1Tx, l2txs []
 	// TBD if ExitTree is only in memory or stored in disk, for the moment
 	// only needed in memory
 	if s.typ == TypeSynchronizer || s.typ == TypeBatchBuilder {
-		exitTree, err = merkletree.NewMerkleTree(memory.NewMemoryStorage(), s.mt.MaxLevels())
+		tmpDir, err := ioutil.TempDir("", "hermez-statedb-exittree")
+		if err != nil {
+			return nil, nil, err
+		}
+		defer func() {
+			if err := os.RemoveAll(tmpDir); err != nil {
+				log.Errorw("Deleting statedb temp exit tree", "err", err)
+			}
+		}()
+		sto, err := pebble.NewPebbleStorage(tmpDir, false)
+		if err != nil {
+			return nil, nil, err
+		}
+		exitTree, err = merkletree.NewMerkleTree(sto, s.mt.MaxLevels())
 		if err != nil {
 			return nil, nil, err
 		}

db/statedb/txprocessors_test.go

@@ -2,6 +2,7 @@ package statedb
 
 import (
 	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/hermeznetwork/hermez-node/common"
@@ -17,6 +18,7 @@ func TestProcessTxsSynchronizer(t *testing.T) {
 
 	dir, err := ioutil.TempDir("", "tmpdb")
 	require.Nil(t, err)
+	defer assert.Nil(t, os.RemoveAll(dir))
 
 	sdb, err := NewStateDB(dir, TypeSynchronizer, 32)
 	assert.Nil(t, err)
@@ -81,6 +83,7 @@ WIP
 func TestProcessTxsBatchBuilder(t *testing.T) {
 	dir, err := ioutil.TempDir("", "tmpdb")
 	require.Nil(t, err)
+	defer assert.Nil(t, os.RemoveAll(dir))
 
 	sdb, err := NewStateDB(dir, TypeBatchBuilder, 32)
 	assert.Nil(t, err)
@@ -130,6 +133,7 @@ func TestProcessTxsBatchBuilder(t *testing.T) {
 func TestZKInputsGeneration(t *testing.T) {
 	dir, err := ioutil.TempDir("", "tmpdb")
 	require.Nil(t, err)
+	defer assert.Nil(t, os.RemoveAll(dir))
 
 	sdb, err := NewStateDB(dir, TypeBatchBuilder, 32)
 	assert.Nil(t, err)

db/statedb/utils_test.go

@@ -3,6 +3,7 @@ package statedb
 import (
 	"io/ioutil"
 	"math/big"
+	"os"
 	"testing"
 
 	ethCommon "github.com/ethereum/go-ethereum/common"
@@ -15,6 +16,7 @@ import (
 func TestGetIdx(t *testing.T) {
 	dir, err := ioutil.TempDir("", "tmpdb")
 	require.Nil(t, err)
+	defer assert.Nil(t, os.RemoveAll(dir))
 
 	sdb, err := NewStateDB(dir, TypeTxSelector, 0)
 	assert.Nil(t, err)