Browse Source

Add apitypes to avoid parsing from/to DB

feature/sql-semaphore1
Arnau B 3 years ago
parent
commit
7c8f380637
9 changed files with 538 additions and 2 deletions
  1. +1
    -0
      api/swagger.yml
  2. +207
    -0
      apitypes/apitypes.go
  3. +286
    -0
      apitypes/apitypes_test.go
  4. +1
    -1
      common/batch.go
  5. +9
    -0
      db/historydb/historydb.go
  6. +22
    -0
      db/historydb/views.go
  7. +2
    -1
      db/migrations/0001.sql
  8. +8
    -0
      db/utils.go
  9. +2
    -0
      go.mod

+ 1
- 0
api/swagger.yml

@ -1492,6 +1492,7 @@ components:
type: string
description: BigInt is an integer encoded as a string for numbers that are very large.
example: "8708856933496328593"
pattern: "^[0-9]$"
FeeSelector:
type: integer
description: Index of the fee type to select, more info [here](https://idocs.hermez.io/#/spec/zkrollup/fee-table?id=transaction-fee-table).

+ 207
- 0
apitypes/apitypes.go

@ -0,0 +1,207 @@
package apitypes
import (
"database/sql/driver"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"math/big"
"strings"
ethCommon "github.com/ethereum/go-ethereum/common"
"github.com/hermeznetwork/hermez-node/common"
"github.com/iden3/go-iden3-crypto/babyjub"
)
// BigIntStr is used to scan/value *big.Int directly into strings from/to sql DBs.
// It assumes that *big.Int are inserted/fetched to/from the DB using the BigIntMeddler meddler
// defined at github.com/hermeznetwork/hermez-node/db
type BigIntStr string
// NewBigIntStr creates a *BigIntStr from a *big.Int.
// If the provided bigInt is nil the returned *BigIntStr will also be nil
func NewBigIntStr(bigInt *big.Int) *BigIntStr {
if bigInt == nil {
return nil
}
bigIntStr := BigIntStr(bigInt.String())
return &bigIntStr
}
// Scan implements Scanner for database/sql
func (b *BigIntStr) Scan(src interface{}) error {
// decode base64 src
var decoded []byte
var err error
if srcStr, ok := src.(string); ok {
// src is a string
decoded, err = base64.StdEncoding.DecodeString(srcStr)
} else if srcBytes, ok := src.([]byte); ok {
// src is []byte
decoded, err = base64.StdEncoding.DecodeString(string(srcBytes))
} else {
// unexpected src
return fmt.Errorf("can't scan %T into apitypes.BigIntStr", src)
}
if err != nil {
return err
}
// decoded bytes to *big.Int
bigInt := &big.Int{}
bigInt = bigInt.SetBytes(decoded)
// *big.Int to BigIntStr
bigIntStr := NewBigIntStr(bigInt)
if bigIntStr == nil {
return nil
}
*b = *bigIntStr
return nil
}
// Value implements valuer for database/sql
func (b BigIntStr) Value() (driver.Value, error) {
// string to *big.Int
bigInt := &big.Int{}
bigInt, ok := bigInt.SetString(string(b), 10)
if !ok || bigInt == nil {
return nil, errors.New("invalid representation of a *big.Int")
}
// *big.Int to base64
return base64.StdEncoding.EncodeToString(bigInt.Bytes()), nil
}
type CollectedFees map[common.TokenID]BigIntStr
func (c *CollectedFees) UnmarshalJSON(text []byte) error {
fmt.Println(string(text))
bigIntMap := make(map[common.TokenID]*big.Int)
if err := json.Unmarshal(text, &bigIntMap); err != nil {
return err
}
bStrMap := make(map[common.TokenID]BigIntStr)
for k, v := range bigIntMap {
bStr := NewBigIntStr(v)
bStrMap[k] = *bStr
}
*c = CollectedFees(bStrMap)
return nil
// fmt.Println(string(text))
// *b = BigIntStr(string(text))
// return nil
// bigInt := &big.Int{}
// if err := bigInt.UnmarshalText(text); err != nil {
// return err
// }
// bigIntStr := NewBigIntStr(bigInt)
// if bigIntStr == nil {
// return nil
// }
// *b = *bigIntStr
// return nil
}
// HezEthAddr is used to scan/value Ethereum Address directly into strings that follow the Ethereum address hez fotmat (^hez:0x[a-fA-F0-9]{40}$) from/to sql DBs.
// It assumes that Ethereum Address are inserted/fetched to/from the DB using the default Scan/Value interface
type HezEthAddr string
// NewHezEthAddr creates a HezEthAddr from an Ethereum addr
func NewHezEthAddr(addr ethCommon.Address) HezEthAddr {
return HezEthAddr("hez:" + addr.String())
}
// ToEthAddr returns an Ethereum Address created from HezEthAddr
func (a HezEthAddr) ToEthAddr() (ethCommon.Address, error) {
addrStr := strings.TrimPrefix(string(a), "hez:")
var addr ethCommon.Address
return addr, addr.UnmarshalText([]byte(addrStr))
}
// Scan implements Scanner for database/sql
func (a *HezEthAddr) Scan(src interface{}) error {
ethAddr := &ethCommon.Address{}
if err := ethAddr.Scan(src); err != nil {
return err
}
if ethAddr == nil {
return nil
}
*a = NewHezEthAddr(*ethAddr)
return nil
}
// Value implements valuer for database/sql
func (a HezEthAddr) Value() (driver.Value, error) {
ethAddr, err := a.ToEthAddr()
if err != nil {
return nil, err
}
return ethAddr.Value()
}
// HezBJJ is used to scan/value *babyjub.PublicKey directly into strings that follow the BJJ public key hez fotmat (^hez:[A-Za-z0-9_-]{44}$) from/to sql DBs.
// It assumes that *babyjub.PublicKey are inserted/fetched to/from the DB using the default Scan/Value interface
type HezBJJ string
// NewHezBJJ creates a HezBJJ from a *babyjub.PublicKey.
// Calling this method with a nil bjj causes panic
func NewHezBJJ(bjj *babyjub.PublicKey) HezBJJ {
pkComp := [32]byte(bjj.Compress())
sum := pkComp[0]
for i := 1; i < len(pkComp); i++ {
sum += pkComp[i]
}
bjjSum := append(pkComp[:], sum)
return HezBJJ("hez:" + base64.RawURLEncoding.EncodeToString(bjjSum))
}
// ToBJJ returns a *babyjub.PublicKey created from HezBJJ
func (b HezBJJ) ToBJJ() (*babyjub.PublicKey, error) {
const decodedLen = 33
const encodedLen = 44
formatErr := errors.New("invalid BJJ format. Must follow this regex: ^hez:[A-Za-z0-9_-]{44}$")
encoded := strings.TrimPrefix(string(b), "hez:")
if len(encoded) != encodedLen {
return nil, formatErr
}
decoded, err := base64.RawURLEncoding.DecodeString(encoded)
if err != nil {
return nil, formatErr
}
if len(decoded) != decodedLen {
return nil, formatErr
}
bjjBytes := [decodedLen - 1]byte{}
copy(bjjBytes[:decodedLen-1], decoded[:decodedLen-1])
sum := bjjBytes[0]
for i := 1; i < len(bjjBytes); i++ {
sum += bjjBytes[i]
}
if decoded[decodedLen-1] != sum {
return nil, errors.New("checksum verification failed")
}
bjjComp := babyjub.PublicKeyComp(bjjBytes)
return bjjComp.Decompress()
}
// Scan implements Scanner for database/sql
func (b *HezBJJ) Scan(src interface{}) error {
bjj := &babyjub.PublicKey{}
if err := bjj.Scan(src); err != nil {
return err
}
if bjj == nil {
return nil
}
*b = NewHezBJJ(bjj)
return nil
}
// Value implements valuer for database/sql
func (b HezBJJ) Value() (driver.Value, error) {
bjj, err := b.ToBJJ()
if err != nil {
return nil, err
}
return bjj.Value()
}

+ 286
- 0
apitypes/apitypes_test.go

@ -0,0 +1,286 @@
package apitypes
import (
"database/sql"
"io/ioutil"
"math/big"
"os"
"testing"
"github.com/iden3/go-iden3-crypto/babyjub"
ethCommon "github.com/ethereum/go-ethereum/common"
dbUtils "github.com/hermeznetwork/hermez-node/db"
_ "github.com/mattn/go-sqlite3" // sqlite driver
"github.com/russross/meddler"
"github.com/stretchr/testify/assert"
)
var db *sql.DB
func TestMain(m *testing.M) {
// Register meddler
meddler.Default = meddler.SQLite
meddler.Register("bigint", dbUtils.BigIntMeddler{})
meddler.Register("bigintnull", dbUtils.BigIntNullMeddler{})
// Create temporary sqlite DB
dir, err := ioutil.TempDir("", "db")
if err != nil {
panic(err)
}
db, err = sql.Open("sqlite3", dir+"sqlite.db")
defer os.RemoveAll(dir)
if err != nil {
panic(err)
}
schema := `CREATE TABLE test (i BLOB);`
if _, err := db.Exec(schema); err != nil {
panic(err)
}
// Run tests
result := m.Run()
os.Exit(result)
}
func TestBigIntStrScannerValuer(t *testing.T) {
// Clean DB
_, err := db.Exec("delete from test")
assert.NoError(t, err)
// Example structs
type bigInMeddlerStruct struct {
I *big.Int `meddler:"i,bigint"` // note the bigint that instructs meddler to use BigIntMeddler
}
type bigIntStrStruct struct {
I BigIntStr `meddler:"i"` // note that no meddler is specified, and Scan/Value will be used
}
type bigInMeddlerStructNil struct {
I *big.Int `meddler:"i,bigintnull"` // note the bigint that instructs meddler to use BigIntNullMeddler
}
type bigIntStrStructNil struct {
I *BigIntStr `meddler:"i"` // note that no meddler is specified, and Scan/Value will be used
}
// Not nil case
// Insert into DB using meddler
const x = int64(12345)
fromMeddler := bigInMeddlerStruct{
I: big.NewInt(x),
}
err = meddler.Insert(db, "test", &fromMeddler)
assert.NoError(t, err)
// Read from DB using BigIntStr
toBigIntStr := bigIntStrStruct{}
err = meddler.QueryRow(db, &toBigIntStr, "select * from test")
assert.NoError(t, err)
assert.Equal(t, fromMeddler.I.String(), string(toBigIntStr.I))
// Clean DB
_, err = db.Exec("delete from test")
assert.NoError(t, err)
// Insert into DB using BigIntStr
fromBigIntStr := bigIntStrStruct{
I: "54321",
}
err = meddler.Insert(db, "test", &fromBigIntStr)
assert.NoError(t, err)
// Read from DB using meddler
toMeddler := bigInMeddlerStruct{}
err = meddler.QueryRow(db, &toMeddler, "select * from test")
assert.NoError(t, err)
assert.Equal(t, string(fromBigIntStr.I), toMeddler.I.String())
// Nil case
// Clean DB
_, err = db.Exec("delete from test")
assert.NoError(t, err)
// Insert into DB using meddler
fromMeddlerNil := bigInMeddlerStructNil{
I: nil,
}
err = meddler.Insert(db, "test", &fromMeddlerNil)
assert.NoError(t, err)
// Read from DB using BigIntStr
foo := BigIntStr("foo")
toBigIntStrNil := bigIntStrStructNil{
I: &foo, // check that this will be set to nil, not because of not being initialized
}
err = meddler.QueryRow(db, &toBigIntStrNil, "select * from test")
assert.NoError(t, err)
assert.Nil(t, toBigIntStrNil.I)
// Clean DB
_, err = db.Exec("delete from test")
assert.NoError(t, err)
// Insert into DB using BigIntStr
fromBigIntStrNil := bigIntStrStructNil{
I: nil,
}
err = meddler.Insert(db, "test", &fromBigIntStrNil)
assert.NoError(t, err)
// Read from DB using meddler
toMeddlerNil := bigInMeddlerStructNil{
I: big.NewInt(x), // check that this will be set to nil, not because of not being initialized
}
err = meddler.QueryRow(db, &toMeddlerNil, "select * from test")
assert.NoError(t, err)
assert.Nil(t, toMeddlerNil.I)
}
func TestHezEthAddr(t *testing.T) {
// Clean DB
_, err := db.Exec("delete from test")
assert.NoError(t, err)
// Example structs
type ethAddrStruct struct {
I ethCommon.Address `meddler:"i"`
}
type hezEthAddrStruct struct {
I HezEthAddr `meddler:"i"`
}
type ethAddrStructNil struct {
I *ethCommon.Address `meddler:"i"`
}
type hezEthAddrStructNil struct {
I *HezEthAddr `meddler:"i"`
}
// Not nil case
// Insert into DB using ethCommon.Address Scan/Value
fromEth := ethAddrStruct{
I: ethCommon.BigToAddress(big.NewInt(73737373)),
}
err = meddler.Insert(db, "test", &fromEth)
assert.NoError(t, err)
// Read from DB using HezEthAddr Scan/Value
toHezEth := hezEthAddrStruct{}
err = meddler.QueryRow(db, &toHezEth, "select * from test")
assert.NoError(t, err)
assert.Equal(t, NewHezEthAddr(fromEth.I), toHezEth.I)
// Clean DB
_, err = db.Exec("delete from test")
assert.NoError(t, err)
// Insert into DB using HezEthAddr Scan/Value
fromHezEth := hezEthAddrStruct{
I: NewHezEthAddr(ethCommon.BigToAddress(big.NewInt(3786872586))),
}
err = meddler.Insert(db, "test", &fromHezEth)
assert.NoError(t, err)
// Read from DB using ethCommon.Address Scan/Value
toEth := ethAddrStruct{}
err = meddler.QueryRow(db, &toEth, "select * from test")
assert.NoError(t, err)
assert.Equal(t, fromHezEth.I, NewHezEthAddr(toEth.I))
// Nil case
// Clean DB
_, err = db.Exec("delete from test")
assert.NoError(t, err)
// Insert into DB using ethCommon.Address Scan/Value
fromEthNil := ethAddrStructNil{
I: nil,
}
err = meddler.Insert(db, "test", &fromEthNil)
assert.NoError(t, err)
// Read from DB using HezEthAddr Scan/Value
foo := HezEthAddr("foo")
toHezEthNil := hezEthAddrStructNil{
I: &foo, // check that this will be set to nil, not because of not being initialized
}
err = meddler.QueryRow(db, &toHezEthNil, "select * from test")
assert.NoError(t, err)
assert.Nil(t, toHezEthNil.I)
// Clean DB
_, err = db.Exec("delete from test")
assert.NoError(t, err)
// Insert into DB using HezEthAddr Scan/Value
fromHezEthNil := hezEthAddrStructNil{
I: nil,
}
err = meddler.Insert(db, "test", &fromHezEthNil)
assert.NoError(t, err)
// Read from DB using ethCommon.Address Scan/Value
fooAddr := ethCommon.BigToAddress(big.NewInt(1))
toEthNil := ethAddrStructNil{
I: &fooAddr, // check that this will be set to nil, not because of not being initialized
}
err = meddler.QueryRow(db, &toEthNil, "select * from test")
assert.NoError(t, err)
assert.Nil(t, toEthNil.I)
}
func TestHezBJJ(t *testing.T) {
// Clean DB
_, err := db.Exec("delete from test")
assert.NoError(t, err)
// Example structs
type bjjStruct struct {
I *babyjub.PublicKey `meddler:"i"`
}
type hezBJJStruct struct {
I HezBJJ `meddler:"i"`
}
type hezBJJStructNil struct {
I *HezBJJ `meddler:"i"`
}
// Not nil case
// Insert into DB using *babyjub.PublicKey Scan/Value
priv := babyjub.NewRandPrivKey()
fromBJJ := bjjStruct{
I: priv.Public(),
}
err = meddler.Insert(db, "test", &fromBJJ)
assert.NoError(t, err)
// Read from DB using HezBJJ Scan/Value
toHezBJJ := hezBJJStruct{}
err = meddler.QueryRow(db, &toHezBJJ, "select * from test")
assert.NoError(t, err)
assert.Equal(t, NewHezBJJ(fromBJJ.I), toHezBJJ.I)
// Clean DB
_, err = db.Exec("delete from test")
assert.NoError(t, err)
// Insert into DB using HezBJJ Scan/Value
fromHezBJJ := hezBJJStruct{
I: NewHezBJJ(priv.Public()),
}
err = meddler.Insert(db, "test", &fromHezBJJ)
assert.NoError(t, err)
// Read from DB using *babyjub.PublicKey Scan/Value
toBJJ := bjjStruct{}
err = meddler.QueryRow(db, &toBJJ, "select * from test")
assert.NoError(t, err)
assert.Equal(t, fromHezBJJ.I, NewHezBJJ(toBJJ.I))
// Nil case
// Clean DB
_, err = db.Exec("delete from test")
assert.NoError(t, err)
// Insert into DB using *babyjub.PublicKey Scan/Value
fromBJJNil := bjjStruct{
I: nil,
}
err = meddler.Insert(db, "test", &fromBJJNil)
assert.NoError(t, err)
// Read from DB using HezBJJ Scan/Value
foo := HezBJJ("foo")
toHezBJJNil := hezBJJStructNil{
I: &foo, // check that this will be set to nil, not because of not being initialized
}
err = meddler.QueryRow(db, &toHezBJJNil, "select * from test")
assert.NoError(t, err)
assert.Nil(t, toHezBJJNil.I)
// Clean DB
_, err = db.Exec("delete from test")
assert.NoError(t, err)
// Insert into DB using HezBJJ Scan/Value
fromHezBJJNil := hezBJJStructNil{
I: nil,
}
err = meddler.Insert(db, "test", &fromHezBJJNil)
assert.NoError(t, err)
// Read from DB using *babyjub.PublicKey Scan/Value
toBJJNil := bjjStruct{
I: priv.Public(), // check that this will be set to nil, not because of not being initialized
}
err = meddler.QueryRow(db, &toBJJNil, "select * from test")
assert.NoError(t, err)
assert.Nil(t, toBJJNil.I)
}

+ 1
- 1
common/batch.go

@ -14,7 +14,7 @@ const batchNumBytesLen = 8
type Batch struct {
BatchNum BatchNum `meddler:"batch_num"`
EthBlockNum int64 `meddler:"eth_block_num"` // Ethereum block in which the batch is forged
ForgerAddr ethCommon.Address `meddler:"forger_addr"` // TODO: Should this be retrieved via slot reference?
ForgerAddr ethCommon.Address `meddler:"forger_addr"`
CollectedFees map[TokenID]*big.Int `meddler:"fees_collected,json"`
StateRoot *big.Int `meddler:"state_root,bigint"`
NumAccounts int `meddler:"num_accounts"`

+ 9
- 0
db/historydb/historydb.go

@ -150,6 +150,15 @@ func (hdb *HistoryDB) addBatches(d meddler.DB, batches []common.Batch) error {
return nil
}
// GetBatch return the batch with the given batchNum
func (hdb *HistoryDB) GetBatch(batchNum common.BatchNum) (HistoryBatch, error) {
var batch *common.Batch
return batch, meddler.QueryRow(
hdb.db, &batch,
"SELECT * FROM batch WHERE batch_num == $1;", batchNum,
)
}
// GetBatches retrieve batches from the DB, given a range of batch numbers defined by from and to
func (hdb *HistoryDB) GetBatches(from, to common.BatchNum) ([]common.Batch, error) {
var batches []*common.Batch

+ 22
- 0
db/historydb/views.go

@ -6,6 +6,7 @@ import (
ethCommon "github.com/ethereum/go-ethereum/common"
"github.com/hermeznetwork/hermez-node/common"
"github.com/hermeznetwork/hermez-node/db"
"github.com/iden3/go-iden3-crypto/babyjub"
"github.com/iden3/go-merkletree"
)
@ -131,3 +132,24 @@ type HistoryCoordinator struct {
FirstItem int `meddler:"first_item"`
LastItem int `meddler:"last_item"`
}
// HistoryBatch is a representation of a batch with additional information
// required by the API, and extracted by joining block table
type HistoryBatch struct {
ItemID int `json:"itemId" meddler:"item_id"`
BatchNum common.BatchNum `json:"batchNum" meddler:"batch_num"`
EthBlockNum int64 `json:"ethereumBlockNum" meddler:"eth_block_num"`
EthBlockHash ethCommon.Hash `json:"ethereumBlockHash" meddler:"hash"`
Timestamp time.Time `json:"timestamp" meddler:"timestamp,utctime"`
ForgerAddr ethCommon.Address `json:"forgerAddr" meddler:"forger_addr"`
CollectedFees map[common.TokenID]db.BigIntStr `json:"collectedFees" meddler:"fees_collected,json"`
TotalFeesUSD float64 `json:"historicTotalCollectedFeesUSD" meddler:"total_fees_usd"`
StateRoot db.BigIntStr `json:"stateRoot" meddler:"state_root"`
NumAccounts int `json:"numAccounts" meddler:"num_accounts"`
ExitRoot db.BigIntStr `json:"exitRoot" meddler:"exit_root"`
ForgeL1TxsNum *int64 `json:"forgeL1TransactionsNum" meddler:"forge_l1_txs_num"`
SlotNum int64 `json:"slotNum" meddler:"slot_num"`
TotalItems int `json:"-" meddler:"total_items"`
FirstItem int `json:"-" meddler:"first_item"`
LastItem int `json:"-" meddler:"last_item"`
}

+ 2
- 1
db/migrations/0001.sql

@ -16,7 +16,8 @@ CREATE TABLE coordinator (
);
CREATE TABLE batch (
batch_num BIGINT PRIMARY KEY,
item_id SERIAL PRIMARY KEY,
batch_num BIGINT NOT NULL,
eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
forger_addr BYTEA NOT NULL, -- fake foreign key for coordinator
fees_collected BYTEA NOT NULL,

+ 8
- 0
db/utils.go

@ -192,3 +192,11 @@ type Paginationer interface {
GetPagination() *Pagination
Len() int
}
// BigIntStr is used to Marshal *big.Int directly into strings
type BigIntStr big.Int
func (b BigIntStr) MarshalText() ([]byte, error) {
bigInt := big.Int(b)
return []byte((&bigInt).String()), nil
}

+ 2
- 0
go.mod

@ -4,6 +4,7 @@ go 1.14
require (
github.com/BurntSushi/toml v0.3.1
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015
github.com/dghubble/sling v1.3.0
github.com/ethereum/go-ethereum v1.9.17
github.com/getkin/kin-openapi v0.22.0
@ -16,6 +17,7 @@ require (
github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a
github.com/jmoiron/sqlx v1.2.0
github.com/lib/pq v1.8.0
github.com/mattn/go-sqlite3 v2.0.3+incompatible
github.com/mitchellh/copystructure v1.0.0
github.com/rogpeppe/go-internal v1.6.1 // indirect
github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351

Loading…
Cancel
Save