From 7c8f380637aee4c049454fdc867229f52c9c55e4 Mon Sep 17 00:00:00 2001 From: Arnau B Date: Thu, 22 Oct 2020 17:10:17 +0200 Subject: [PATCH] Add apitypes to avoid parsing from/to DB --- api/swagger.yml | 1 + apitypes/apitypes.go | 207 +++++++++++++++++++++++++++ apitypes/apitypes_test.go | 286 ++++++++++++++++++++++++++++++++++++++ common/batch.go | 2 +- db/historydb/historydb.go | 9 ++ db/historydb/views.go | 22 +++ db/migrations/0001.sql | 3 +- db/utils.go | 8 ++ go.mod | 2 + 9 files changed, 538 insertions(+), 2 deletions(-) create mode 100644 apitypes/apitypes.go create mode 100644 apitypes/apitypes_test.go diff --git a/api/swagger.yml b/api/swagger.yml index 23ec8f5..6cbde7c 100644 --- a/api/swagger.yml +++ b/api/swagger.yml @@ -1492,6 +1492,7 @@ components: type: string description: BigInt is an integer encoded as a string for numbers that are very large. example: "8708856933496328593" + pattern: "^[0-9]$" FeeSelector: type: integer description: Index of the fee type to select, more info [here](https://idocs.hermez.io/#/spec/zkrollup/fee-table?id=transaction-fee-table). diff --git a/apitypes/apitypes.go b/apitypes/apitypes.go new file mode 100644 index 0000000..d64bf82 --- /dev/null +++ b/apitypes/apitypes.go @@ -0,0 +1,207 @@ +package apitypes + +import ( + "database/sql/driver" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "math/big" + "strings" + + ethCommon "github.com/ethereum/go-ethereum/common" + "github.com/hermeznetwork/hermez-node/common" + "github.com/iden3/go-iden3-crypto/babyjub" +) + +// BigIntStr is used to scan/value *big.Int directly into strings from/to sql DBs. +// It assumes that *big.Int are inserted/fetched to/from the DB using the BigIntMeddler meddler +// defined at github.com/hermeznetwork/hermez-node/db +type BigIntStr string + +// NewBigIntStr creates a *BigIntStr from a *big.Int. +// If the provided bigInt is nil the returned *BigIntStr will also be nil +func NewBigIntStr(bigInt *big.Int) *BigIntStr { + if bigInt == nil { + return nil + } + bigIntStr := BigIntStr(bigInt.String()) + return &bigIntStr +} + +// Scan implements Scanner for database/sql +func (b *BigIntStr) Scan(src interface{}) error { + // decode base64 src + var decoded []byte + var err error + if srcStr, ok := src.(string); ok { + // src is a string + decoded, err = base64.StdEncoding.DecodeString(srcStr) + } else if srcBytes, ok := src.([]byte); ok { + // src is []byte + decoded, err = base64.StdEncoding.DecodeString(string(srcBytes)) + } else { + // unexpected src + return fmt.Errorf("can't scan %T into apitypes.BigIntStr", src) + } + if err != nil { + return err + } + // decoded bytes to *big.Int + bigInt := &big.Int{} + bigInt = bigInt.SetBytes(decoded) + // *big.Int to BigIntStr + bigIntStr := NewBigIntStr(bigInt) + if bigIntStr == nil { + return nil + } + *b = *bigIntStr + return nil +} + +// Value implements valuer for database/sql +func (b BigIntStr) Value() (driver.Value, error) { + // string to *big.Int + bigInt := &big.Int{} + bigInt, ok := bigInt.SetString(string(b), 10) + if !ok || bigInt == nil { + return nil, errors.New("invalid representation of a *big.Int") + } + // *big.Int to base64 + return base64.StdEncoding.EncodeToString(bigInt.Bytes()), nil +} + +type CollectedFees map[common.TokenID]BigIntStr + +func (c *CollectedFees) UnmarshalJSON(text []byte) error { + fmt.Println(string(text)) + bigIntMap := make(map[common.TokenID]*big.Int) + if err := json.Unmarshal(text, &bigIntMap); err != nil { + return err + } + bStrMap := make(map[common.TokenID]BigIntStr) + for k, v := range bigIntMap { + bStr := NewBigIntStr(v) + bStrMap[k] = *bStr + } + *c = CollectedFees(bStrMap) + return nil + // fmt.Println(string(text)) + // *b = BigIntStr(string(text)) + // return nil + // bigInt := &big.Int{} + // if err := bigInt.UnmarshalText(text); err != nil { + // return err + // } + // bigIntStr := NewBigIntStr(bigInt) + // if bigIntStr == nil { + // return nil + // } + // *b = *bigIntStr + // return nil +} + +// HezEthAddr is used to scan/value Ethereum Address directly into strings that follow the Ethereum address hez fotmat (^hez:0x[a-fA-F0-9]{40}$) from/to sql DBs. +// It assumes that Ethereum Address are inserted/fetched to/from the DB using the default Scan/Value interface +type HezEthAddr string + +// NewHezEthAddr creates a HezEthAddr from an Ethereum addr +func NewHezEthAddr(addr ethCommon.Address) HezEthAddr { + return HezEthAddr("hez:" + addr.String()) +} + +// ToEthAddr returns an Ethereum Address created from HezEthAddr +func (a HezEthAddr) ToEthAddr() (ethCommon.Address, error) { + addrStr := strings.TrimPrefix(string(a), "hez:") + var addr ethCommon.Address + return addr, addr.UnmarshalText([]byte(addrStr)) +} + +// Scan implements Scanner for database/sql +func (a *HezEthAddr) Scan(src interface{}) error { + ethAddr := ðCommon.Address{} + if err := ethAddr.Scan(src); err != nil { + return err + } + if ethAddr == nil { + return nil + } + *a = NewHezEthAddr(*ethAddr) + return nil +} + +// Value implements valuer for database/sql +func (a HezEthAddr) Value() (driver.Value, error) { + ethAddr, err := a.ToEthAddr() + if err != nil { + return nil, err + } + return ethAddr.Value() +} + +// HezBJJ is used to scan/value *babyjub.PublicKey directly into strings that follow the BJJ public key hez fotmat (^hez:[A-Za-z0-9_-]{44}$) from/to sql DBs. +// It assumes that *babyjub.PublicKey are inserted/fetched to/from the DB using the default Scan/Value interface +type HezBJJ string + +// NewHezBJJ creates a HezBJJ from a *babyjub.PublicKey. +// Calling this method with a nil bjj causes panic +func NewHezBJJ(bjj *babyjub.PublicKey) HezBJJ { + pkComp := [32]byte(bjj.Compress()) + sum := pkComp[0] + for i := 1; i < len(pkComp); i++ { + sum += pkComp[i] + } + bjjSum := append(pkComp[:], sum) + return HezBJJ("hez:" + base64.RawURLEncoding.EncodeToString(bjjSum)) +} + +// ToBJJ returns a *babyjub.PublicKey created from HezBJJ +func (b HezBJJ) ToBJJ() (*babyjub.PublicKey, error) { + const decodedLen = 33 + const encodedLen = 44 + formatErr := errors.New("invalid BJJ format. Must follow this regex: ^hez:[A-Za-z0-9_-]{44}$") + encoded := strings.TrimPrefix(string(b), "hez:") + if len(encoded) != encodedLen { + return nil, formatErr + } + decoded, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + return nil, formatErr + } + if len(decoded) != decodedLen { + return nil, formatErr + } + bjjBytes := [decodedLen - 1]byte{} + copy(bjjBytes[:decodedLen-1], decoded[:decodedLen-1]) + sum := bjjBytes[0] + for i := 1; i < len(bjjBytes); i++ { + sum += bjjBytes[i] + } + if decoded[decodedLen-1] != sum { + return nil, errors.New("checksum verification failed") + } + bjjComp := babyjub.PublicKeyComp(bjjBytes) + return bjjComp.Decompress() +} + +// Scan implements Scanner for database/sql +func (b *HezBJJ) Scan(src interface{}) error { + bjj := &babyjub.PublicKey{} + if err := bjj.Scan(src); err != nil { + return err + } + if bjj == nil { + return nil + } + *b = NewHezBJJ(bjj) + return nil +} + +// Value implements valuer for database/sql +func (b HezBJJ) Value() (driver.Value, error) { + bjj, err := b.ToBJJ() + if err != nil { + return nil, err + } + return bjj.Value() +} diff --git a/apitypes/apitypes_test.go b/apitypes/apitypes_test.go new file mode 100644 index 0000000..1614111 --- /dev/null +++ b/apitypes/apitypes_test.go @@ -0,0 +1,286 @@ +package apitypes + +import ( + "database/sql" + "io/ioutil" + "math/big" + "os" + "testing" + + "github.com/iden3/go-iden3-crypto/babyjub" + + ethCommon "github.com/ethereum/go-ethereum/common" + dbUtils "github.com/hermeznetwork/hermez-node/db" + _ "github.com/mattn/go-sqlite3" // sqlite driver + "github.com/russross/meddler" + "github.com/stretchr/testify/assert" +) + +var db *sql.DB + +func TestMain(m *testing.M) { + // Register meddler + meddler.Default = meddler.SQLite + meddler.Register("bigint", dbUtils.BigIntMeddler{}) + meddler.Register("bigintnull", dbUtils.BigIntNullMeddler{}) + // Create temporary sqlite DB + dir, err := ioutil.TempDir("", "db") + if err != nil { + panic(err) + } + db, err = sql.Open("sqlite3", dir+"sqlite.db") + defer os.RemoveAll(dir) + if err != nil { + panic(err) + } + schema := `CREATE TABLE test (i BLOB);` + if _, err := db.Exec(schema); err != nil { + panic(err) + } + // Run tests + result := m.Run() + os.Exit(result) +} + +func TestBigIntStrScannerValuer(t *testing.T) { + // Clean DB + _, err := db.Exec("delete from test") + assert.NoError(t, err) + // Example structs + type bigInMeddlerStruct struct { + I *big.Int `meddler:"i,bigint"` // note the bigint that instructs meddler to use BigIntMeddler + } + type bigIntStrStruct struct { + I BigIntStr `meddler:"i"` // note that no meddler is specified, and Scan/Value will be used + } + type bigInMeddlerStructNil struct { + I *big.Int `meddler:"i,bigintnull"` // note the bigint that instructs meddler to use BigIntNullMeddler + } + type bigIntStrStructNil struct { + I *BigIntStr `meddler:"i"` // note that no meddler is specified, and Scan/Value will be used + } + + // Not nil case + // Insert into DB using meddler + const x = int64(12345) + fromMeddler := bigInMeddlerStruct{ + I: big.NewInt(x), + } + err = meddler.Insert(db, "test", &fromMeddler) + assert.NoError(t, err) + // Read from DB using BigIntStr + toBigIntStr := bigIntStrStruct{} + err = meddler.QueryRow(db, &toBigIntStr, "select * from test") + assert.NoError(t, err) + assert.Equal(t, fromMeddler.I.String(), string(toBigIntStr.I)) + // Clean DB + _, err = db.Exec("delete from test") + assert.NoError(t, err) + // Insert into DB using BigIntStr + fromBigIntStr := bigIntStrStruct{ + I: "54321", + } + err = meddler.Insert(db, "test", &fromBigIntStr) + assert.NoError(t, err) + // Read from DB using meddler + toMeddler := bigInMeddlerStruct{} + err = meddler.QueryRow(db, &toMeddler, "select * from test") + assert.NoError(t, err) + assert.Equal(t, string(fromBigIntStr.I), toMeddler.I.String()) + + // Nil case + // Clean DB + _, err = db.Exec("delete from test") + assert.NoError(t, err) + // Insert into DB using meddler + fromMeddlerNil := bigInMeddlerStructNil{ + I: nil, + } + err = meddler.Insert(db, "test", &fromMeddlerNil) + assert.NoError(t, err) + // Read from DB using BigIntStr + foo := BigIntStr("foo") + toBigIntStrNil := bigIntStrStructNil{ + I: &foo, // check that this will be set to nil, not because of not being initialized + } + err = meddler.QueryRow(db, &toBigIntStrNil, "select * from test") + assert.NoError(t, err) + assert.Nil(t, toBigIntStrNil.I) + // Clean DB + _, err = db.Exec("delete from test") + assert.NoError(t, err) + // Insert into DB using BigIntStr + fromBigIntStrNil := bigIntStrStructNil{ + I: nil, + } + err = meddler.Insert(db, "test", &fromBigIntStrNil) + assert.NoError(t, err) + // Read from DB using meddler + toMeddlerNil := bigInMeddlerStructNil{ + I: big.NewInt(x), // check that this will be set to nil, not because of not being initialized + } + err = meddler.QueryRow(db, &toMeddlerNil, "select * from test") + assert.NoError(t, err) + assert.Nil(t, toMeddlerNil.I) +} + +func TestHezEthAddr(t *testing.T) { + // Clean DB + _, err := db.Exec("delete from test") + assert.NoError(t, err) + // Example structs + type ethAddrStruct struct { + I ethCommon.Address `meddler:"i"` + } + type hezEthAddrStruct struct { + I HezEthAddr `meddler:"i"` + } + type ethAddrStructNil struct { + I *ethCommon.Address `meddler:"i"` + } + type hezEthAddrStructNil struct { + I *HezEthAddr `meddler:"i"` + } + + // Not nil case + // Insert into DB using ethCommon.Address Scan/Value + fromEth := ethAddrStruct{ + I: ethCommon.BigToAddress(big.NewInt(73737373)), + } + err = meddler.Insert(db, "test", &fromEth) + assert.NoError(t, err) + // Read from DB using HezEthAddr Scan/Value + toHezEth := hezEthAddrStruct{} + err = meddler.QueryRow(db, &toHezEth, "select * from test") + assert.NoError(t, err) + assert.Equal(t, NewHezEthAddr(fromEth.I), toHezEth.I) + // Clean DB + _, err = db.Exec("delete from test") + assert.NoError(t, err) + // Insert into DB using HezEthAddr Scan/Value + fromHezEth := hezEthAddrStruct{ + I: NewHezEthAddr(ethCommon.BigToAddress(big.NewInt(3786872586))), + } + err = meddler.Insert(db, "test", &fromHezEth) + assert.NoError(t, err) + // Read from DB using ethCommon.Address Scan/Value + toEth := ethAddrStruct{} + err = meddler.QueryRow(db, &toEth, "select * from test") + assert.NoError(t, err) + assert.Equal(t, fromHezEth.I, NewHezEthAddr(toEth.I)) + + // Nil case + // Clean DB + _, err = db.Exec("delete from test") + assert.NoError(t, err) + // Insert into DB using ethCommon.Address Scan/Value + fromEthNil := ethAddrStructNil{ + I: nil, + } + err = meddler.Insert(db, "test", &fromEthNil) + assert.NoError(t, err) + // Read from DB using HezEthAddr Scan/Value + foo := HezEthAddr("foo") + toHezEthNil := hezEthAddrStructNil{ + I: &foo, // check that this will be set to nil, not because of not being initialized + } + err = meddler.QueryRow(db, &toHezEthNil, "select * from test") + assert.NoError(t, err) + assert.Nil(t, toHezEthNil.I) + // Clean DB + _, err = db.Exec("delete from test") + assert.NoError(t, err) + // Insert into DB using HezEthAddr Scan/Value + fromHezEthNil := hezEthAddrStructNil{ + I: nil, + } + err = meddler.Insert(db, "test", &fromHezEthNil) + assert.NoError(t, err) + // Read from DB using ethCommon.Address Scan/Value + fooAddr := ethCommon.BigToAddress(big.NewInt(1)) + toEthNil := ethAddrStructNil{ + I: &fooAddr, // check that this will be set to nil, not because of not being initialized + } + err = meddler.QueryRow(db, &toEthNil, "select * from test") + assert.NoError(t, err) + assert.Nil(t, toEthNil.I) +} + +func TestHezBJJ(t *testing.T) { + // Clean DB + _, err := db.Exec("delete from test") + assert.NoError(t, err) + // Example structs + type bjjStruct struct { + I *babyjub.PublicKey `meddler:"i"` + } + type hezBJJStruct struct { + I HezBJJ `meddler:"i"` + } + type hezBJJStructNil struct { + I *HezBJJ `meddler:"i"` + } + + // Not nil case + // Insert into DB using *babyjub.PublicKey Scan/Value + priv := babyjub.NewRandPrivKey() + fromBJJ := bjjStruct{ + I: priv.Public(), + } + err = meddler.Insert(db, "test", &fromBJJ) + assert.NoError(t, err) + // Read from DB using HezBJJ Scan/Value + toHezBJJ := hezBJJStruct{} + err = meddler.QueryRow(db, &toHezBJJ, "select * from test") + assert.NoError(t, err) + assert.Equal(t, NewHezBJJ(fromBJJ.I), toHezBJJ.I) + // Clean DB + _, err = db.Exec("delete from test") + assert.NoError(t, err) + // Insert into DB using HezBJJ Scan/Value + fromHezBJJ := hezBJJStruct{ + I: NewHezBJJ(priv.Public()), + } + err = meddler.Insert(db, "test", &fromHezBJJ) + assert.NoError(t, err) + // Read from DB using *babyjub.PublicKey Scan/Value + toBJJ := bjjStruct{} + err = meddler.QueryRow(db, &toBJJ, "select * from test") + assert.NoError(t, err) + assert.Equal(t, fromHezBJJ.I, NewHezBJJ(toBJJ.I)) + + // Nil case + // Clean DB + _, err = db.Exec("delete from test") + assert.NoError(t, err) + // Insert into DB using *babyjub.PublicKey Scan/Value + fromBJJNil := bjjStruct{ + I: nil, + } + err = meddler.Insert(db, "test", &fromBJJNil) + assert.NoError(t, err) + // Read from DB using HezBJJ Scan/Value + foo := HezBJJ("foo") + toHezBJJNil := hezBJJStructNil{ + I: &foo, // check that this will be set to nil, not because of not being initialized + } + err = meddler.QueryRow(db, &toHezBJJNil, "select * from test") + assert.NoError(t, err) + assert.Nil(t, toHezBJJNil.I) + // Clean DB + _, err = db.Exec("delete from test") + assert.NoError(t, err) + // Insert into DB using HezBJJ Scan/Value + fromHezBJJNil := hezBJJStructNil{ + I: nil, + } + err = meddler.Insert(db, "test", &fromHezBJJNil) + assert.NoError(t, err) + // Read from DB using *babyjub.PublicKey Scan/Value + toBJJNil := bjjStruct{ + I: priv.Public(), // check that this will be set to nil, not because of not being initialized + } + err = meddler.QueryRow(db, &toBJJNil, "select * from test") + assert.NoError(t, err) + assert.Nil(t, toBJJNil.I) +} diff --git a/common/batch.go b/common/batch.go index d132442..1d7206f 100644 --- a/common/batch.go +++ b/common/batch.go @@ -14,7 +14,7 @@ const batchNumBytesLen = 8 type Batch struct { BatchNum BatchNum `meddler:"batch_num"` EthBlockNum int64 `meddler:"eth_block_num"` // Ethereum block in which the batch is forged - ForgerAddr ethCommon.Address `meddler:"forger_addr"` // TODO: Should this be retrieved via slot reference? + ForgerAddr ethCommon.Address `meddler:"forger_addr"` CollectedFees map[TokenID]*big.Int `meddler:"fees_collected,json"` StateRoot *big.Int `meddler:"state_root,bigint"` NumAccounts int `meddler:"num_accounts"` diff --git a/db/historydb/historydb.go b/db/historydb/historydb.go index 7421d02..ae3e831 100644 --- a/db/historydb/historydb.go +++ b/db/historydb/historydb.go @@ -150,6 +150,15 @@ func (hdb *HistoryDB) addBatches(d meddler.DB, batches []common.Batch) error { return nil } +// GetBatch return the batch with the given batchNum +func (hdb *HistoryDB) GetBatch(batchNum common.BatchNum) (HistoryBatch, error) { + var batch *common.Batch + return batch, meddler.QueryRow( + hdb.db, &batch, + "SELECT * FROM batch WHERE batch_num == $1;", batchNum, + ) +} + // GetBatches retrieve batches from the DB, given a range of batch numbers defined by from and to func (hdb *HistoryDB) GetBatches(from, to common.BatchNum) ([]common.Batch, error) { var batches []*common.Batch diff --git a/db/historydb/views.go b/db/historydb/views.go index cdff84c..e29a011 100644 --- a/db/historydb/views.go +++ b/db/historydb/views.go @@ -6,6 +6,7 @@ import ( ethCommon "github.com/ethereum/go-ethereum/common" "github.com/hermeznetwork/hermez-node/common" + "github.com/hermeznetwork/hermez-node/db" "github.com/iden3/go-iden3-crypto/babyjub" "github.com/iden3/go-merkletree" ) @@ -131,3 +132,24 @@ type HistoryCoordinator struct { FirstItem int `meddler:"first_item"` LastItem int `meddler:"last_item"` } + +// HistoryBatch is a representation of a batch with additional information +// required by the API, and extracted by joining block table +type HistoryBatch struct { + ItemID int `json:"itemId" meddler:"item_id"` + BatchNum common.BatchNum `json:"batchNum" meddler:"batch_num"` + EthBlockNum int64 `json:"ethereumBlockNum" meddler:"eth_block_num"` + EthBlockHash ethCommon.Hash `json:"ethereumBlockHash" meddler:"hash"` + Timestamp time.Time `json:"timestamp" meddler:"timestamp,utctime"` + ForgerAddr ethCommon.Address `json:"forgerAddr" meddler:"forger_addr"` + CollectedFees map[common.TokenID]db.BigIntStr `json:"collectedFees" meddler:"fees_collected,json"` + TotalFeesUSD float64 `json:"historicTotalCollectedFeesUSD" meddler:"total_fees_usd"` + StateRoot db.BigIntStr `json:"stateRoot" meddler:"state_root"` + NumAccounts int `json:"numAccounts" meddler:"num_accounts"` + ExitRoot db.BigIntStr `json:"exitRoot" meddler:"exit_root"` + ForgeL1TxsNum *int64 `json:"forgeL1TransactionsNum" meddler:"forge_l1_txs_num"` + SlotNum int64 `json:"slotNum" meddler:"slot_num"` + TotalItems int `json:"-" meddler:"total_items"` + FirstItem int `json:"-" meddler:"first_item"` + LastItem int `json:"-" meddler:"last_item"` +} diff --git a/db/migrations/0001.sql b/db/migrations/0001.sql index 8083d64..05eca3e 100644 --- a/db/migrations/0001.sql +++ b/db/migrations/0001.sql @@ -16,7 +16,8 @@ CREATE TABLE coordinator ( ); CREATE TABLE batch ( - batch_num BIGINT PRIMARY KEY, + item_id SERIAL PRIMARY KEY, + batch_num BIGINT NOT NULL, eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE, forger_addr BYTEA NOT NULL, -- fake foreign key for coordinator fees_collected BYTEA NOT NULL, diff --git a/db/utils.go b/db/utils.go index cb96172..3cb52a0 100644 --- a/db/utils.go +++ b/db/utils.go @@ -192,3 +192,11 @@ type Paginationer interface { GetPagination() *Pagination Len() int } + +// BigIntStr is used to Marshal *big.Int directly into strings +type BigIntStr big.Int + +func (b BigIntStr) MarshalText() ([]byte, error) { + bigInt := big.Int(b) + return []byte((&bigInt).String()), nil +} diff --git a/go.mod b/go.mod index e111650..2a53e25 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( github.com/BurntSushi/toml v0.3.1 + github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 github.com/dghubble/sling v1.3.0 github.com/ethereum/go-ethereum v1.9.17 github.com/getkin/kin-openapi v0.22.0 @@ -16,6 +17,7 @@ require ( github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/jmoiron/sqlx v1.2.0 github.com/lib/pq v1.8.0 + github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/mitchellh/copystructure v1.0.0 github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351