Browse Source
Merge pull request #540 from hermeznetwork/feature/float40upgrade
Feature/float40upgradefeature/txprocessor-update
Eduard S
3 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 682 additions and 562 deletions
Split View
Diff Options
-
+1 -1api/swagger.yml
-
+2 -2common/ethrollup.go
-
+0 -131common/float16.go
-
+0 -132common/float16_test.go
-
+103 -0common/float40.go
-
+95 -0common/float40_test.go
-
+50 -43common/l1tx.go
-
+80 -36common/l1tx_test.go
-
+20 -10common/l2tx.go
-
+74 -14common/l2tx_test.go
-
+51 -32common/pooll2tx.go
-
+91 -70common/pooll2tx_test.go
-
+9 -6common/zk.go
-
+4 -4db/historydb/historydb_test.go
-
+7 -7eth/rollup.go
-
+2 -2eth/rollup_test.go
-
+2 -2test/ethclient.go
-
+3 -1test/txsets/txhelpers.go
-
+2 -2test/zkproof/flows_test.go
-
+2 -1test/zkproof/zkproof_test.go
-
+21 -2txprocessor/txprocessor.go
-
+7 -10txprocessor/txprocessor_test.go
-
+53 -51txprocessor/zkinputsgen_test.go
-
+3 -3txselector/txselector_test.go
@ -1,131 +0,0 @@ |
|||
// Package common Float16 provides methods to work with Hermez custom half float
|
|||
// precision, 16 bits, codification internally called Float16 has been adopted
|
|||
// to encode large integers. This is done in order to save bits when L2
|
|||
// transactions are published.
|
|||
//nolint:gomnd
|
|||
package common |
|||
|
|||
import ( |
|||
"encoding/binary" |
|||
"errors" |
|||
"math/big" |
|||
|
|||
"github.com/hermeznetwork/tracerr" |
|||
) |
|||
|
|||
var ( |
|||
// ErrRoundingLoss is used when converted big.Int to Float16 causes rounding loss
|
|||
ErrRoundingLoss = errors.New("input value causes rounding loss") |
|||
) |
|||
|
|||
// Float16 represents a float in a 16 bit format
|
|||
type Float16 uint16 |
|||
|
|||
// Bytes return a byte array of length 2 with the Float16 value encoded in BigEndian
|
|||
func (f16 Float16) Bytes() []byte { |
|||
var b [2]byte |
|||
binary.BigEndian.PutUint16(b[:], uint16(f16)) |
|||
return b[:] |
|||
} |
|||
|
|||
// Float16FromBytes returns a Float16 from a byte array of 2 bytes.
|
|||
func Float16FromBytes(b []byte) *Float16 { |
|||
f16 := Float16(binary.BigEndian.Uint16(b[:2])) |
|||
return &f16 |
|||
} |
|||
|
|||
// BigInt converts the Float16 to a *big.Int integer
|
|||
func (f16 *Float16) BigInt() *big.Int { |
|||
fl := int64(*f16) |
|||
|
|||
m := big.NewInt(fl & 0x3FF) |
|||
e := big.NewInt(fl >> 11) |
|||
e5 := (fl >> 10) & 0x01 |
|||
|
|||
exp := big.NewInt(0).Exp(big.NewInt(10), e, nil) |
|||
res := m.Mul(m, exp) |
|||
|
|||
if e5 != 0 && e.Cmp(big.NewInt(0)) != 0 { |
|||
res.Add(res, exp.Div(exp, big.NewInt(2))) |
|||
} |
|||
return res |
|||
} |
|||
|
|||
// floorFix2Float converts a fix to a float, always rounding down
|
|||
func floorFix2Float(_f *big.Int) Float16 { |
|||
zero := big.NewInt(0) |
|||
ten := big.NewInt(10) |
|||
e := int64(0) |
|||
|
|||
m := big.NewInt(0) |
|||
m.Set(_f) |
|||
|
|||
if m.Cmp(zero) == 0 { |
|||
return 0 |
|||
} |
|||
|
|||
s := big.NewInt(0).Rsh(m, 10) |
|||
|
|||
for s.Cmp(zero) != 0 { |
|||
m.Div(m, ten) |
|||
s.Rsh(m, 10) |
|||
e++ |
|||
} |
|||
|
|||
return Float16(m.Int64() | e<<11) |
|||
} |
|||
|
|||
// NewFloat16 encodes a *big.Int integer as a Float16, returning error in
|
|||
// case of loss during the encoding.
|
|||
func NewFloat16(f *big.Int) (Float16, error) { |
|||
fl1 := floorFix2Float(f) |
|||
fi1 := fl1.BigInt() |
|||
fl2 := fl1 | 0x400 |
|||
fi2 := fl2.BigInt() |
|||
|
|||
m3 := (fl1 & 0x3FF) + 1 |
|||
e3 := fl1 >> 11 |
|||
|
|||
if m3&0x400 == 0 { |
|||
m3 = 0x66 |
|||
e3++ |
|||
} |
|||
|
|||
fl3 := m3 + e3<<11 |
|||
fi3 := fl3.BigInt() |
|||
|
|||
res := fl1 |
|||
|
|||
d := big.NewInt(0).Abs(fi1.Sub(fi1, f)) |
|||
d2 := big.NewInt(0).Abs(fi2.Sub(fi2, f)) |
|||
|
|||
if d.Cmp(d2) == 1 { |
|||
res = fl2 |
|||
d = d2 |
|||
} |
|||
|
|||
d3 := big.NewInt(0).Abs(fi3.Sub(fi3, f)) |
|||
|
|||
if d.Cmp(d3) == 1 { |
|||
res = fl3 |
|||
} |
|||
|
|||
// Do rounding check
|
|||
if res.BigInt().Cmp(f) == 0 { |
|||
return res, nil |
|||
} |
|||
return res, tracerr.Wrap(ErrRoundingLoss) |
|||
} |
|||
|
|||
// NewFloat16Floor encodes a big.Int integer as a Float16, rounding down in
|
|||
// case of loss during the encoding.
|
|||
func NewFloat16Floor(f *big.Int) Float16 { |
|||
fl1 := floorFix2Float(f) |
|||
fl2 := fl1 | 0x400 |
|||
fi2 := fl2.BigInt() |
|||
|
|||
if fi2.Cmp(f) < 1 { |
|||
return fl2 |
|||
} |
|||
return fl1 |
|||
} |
|||
@ -1,132 +0,0 @@ |
|||
package common |
|||
|
|||
import ( |
|||
"math/big" |
|||
"testing" |
|||
|
|||
"github.com/hermeznetwork/tracerr" |
|||
"github.com/stretchr/testify/assert" |
|||
) |
|||
|
|||
func TestConversions(t *testing.T) { |
|||
testVector := map[Float16]string{ |
|||
0x307B: "123000000", |
|||
0x1DC6: "454500", |
|||
0xFFFF: "10235000000000000000000000000000000", |
|||
0x0000: "0", |
|||
0x0400: "0", |
|||
0x0001: "1", |
|||
0x0401: "1", |
|||
0x0800: "0", |
|||
0x0c00: "5", |
|||
0x0801: "10", |
|||
0x0c01: "15", |
|||
} |
|||
|
|||
for test := range testVector { |
|||
fix := test.BigInt() |
|||
|
|||
assert.Equal(t, fix.String(), testVector[test]) |
|||
|
|||
bi := big.NewInt(0) |
|||
bi.SetString(testVector[test], 10) |
|||
|
|||
fl, err := NewFloat16(bi) |
|||
assert.Equal(t, nil, err) |
|||
|
|||
fx2 := fl.BigInt() |
|||
assert.Equal(t, fx2.String(), testVector[test]) |
|||
} |
|||
} |
|||
|
|||
func TestFloorFix2Float(t *testing.T) { |
|||
testVector := map[string]Float16{ |
|||
"87999990000000000": 0x776f, |
|||
"87950000000000001": 0x776f, |
|||
"87950000000000000": 0x776f, |
|||
"87949999999999999": 0x736f, |
|||
} |
|||
|
|||
for test := range testVector { |
|||
bi := big.NewInt(0) |
|||
bi.SetString(test, 10) |
|||
|
|||
testFloat := NewFloat16Floor(bi) |
|||
|
|||
assert.Equal(t, testFloat, testVector[test]) |
|||
} |
|||
} |
|||
|
|||
func TestConversionLosses(t *testing.T) { |
|||
a := big.NewInt(1000) |
|||
b, err := NewFloat16(a) |
|||
assert.Equal(t, nil, err) |
|||
c := b.BigInt() |
|||
assert.Equal(t, c, a) |
|||
|
|||
a = big.NewInt(1024) |
|||
b, err = NewFloat16(a) |
|||
assert.Equal(t, ErrRoundingLoss, tracerr.Unwrap(err)) |
|||
c = b.BigInt() |
|||
assert.NotEqual(t, c, a) |
|||
|
|||
a = big.NewInt(32767) |
|||
b, err = NewFloat16(a) |
|||
assert.Equal(t, ErrRoundingLoss, tracerr.Unwrap(err)) |
|||
c = b.BigInt() |
|||
assert.NotEqual(t, c, a) |
|||
|
|||
a = big.NewInt(32768) |
|||
b, err = NewFloat16(a) |
|||
assert.Equal(t, ErrRoundingLoss, tracerr.Unwrap(err)) |
|||
c = b.BigInt() |
|||
assert.NotEqual(t, c, a) |
|||
|
|||
a = big.NewInt(65536000) |
|||
b, err = NewFloat16(a) |
|||
assert.Equal(t, ErrRoundingLoss, tracerr.Unwrap(err)) |
|||
c = b.BigInt() |
|||
assert.NotEqual(t, c, a) |
|||
} |
|||
|
|||
func BenchmarkFloat16(b *testing.B) { |
|||
newBigInt := func(s string) *big.Int { |
|||
bigInt, ok := new(big.Int).SetString(s, 10) |
|||
if !ok { |
|||
panic("Bad big int") |
|||
} |
|||
return bigInt |
|||
} |
|||
type pair struct { |
|||
Float16 Float16 |
|||
BigInt *big.Int |
|||
} |
|||
testVector := []pair{ |
|||
{0x307B, newBigInt("123000000")}, |
|||
{0x1DC6, newBigInt("454500")}, |
|||
{0xFFFF, newBigInt("10235000000000000000000000000000000")}, |
|||
{0x0000, newBigInt("0")}, |
|||
{0x0400, newBigInt("0")}, |
|||
{0x0001, newBigInt("1")}, |
|||
{0x0401, newBigInt("1")}, |
|||
{0x0800, newBigInt("0")}, |
|||
{0x0c00, newBigInt("5")}, |
|||
{0x0801, newBigInt("10")}, |
|||
{0x0c01, newBigInt("15")}, |
|||
} |
|||
b.Run("floorFix2Float()", func(b *testing.B) { |
|||
for i := 0; i < b.N; i++ { |
|||
NewFloat16Floor(testVector[i%len(testVector)].BigInt) |
|||
} |
|||
}) |
|||
b.Run("NewFloat16()", func(b *testing.B) { |
|||
for i := 0; i < b.N; i++ { |
|||
_, _ = NewFloat16(testVector[i%len(testVector)].BigInt) |
|||
} |
|||
}) |
|||
b.Run("Float16.BigInt()", func(b *testing.B) { |
|||
for i := 0; i < b.N; i++ { |
|||
testVector[i%len(testVector)].Float16.BigInt() |
|||
} |
|||
}) |
|||
} |
|||
@ -0,0 +1,103 @@ |
|||
// Package common Float40 provides methods to work with Hermez custom half
|
|||
// float precision, 40 bits, codification internally called Float40 has been
|
|||
// adopted to encode large integers. This is done in order to save bits when L2
|
|||
// transactions are published.
|
|||
//nolint:gomnd
|
|||
package common |
|||
|
|||
import ( |
|||
"encoding/binary" |
|||
"errors" |
|||
"math/big" |
|||
|
|||
"github.com/hermeznetwork/tracerr" |
|||
) |
|||
|
|||
const ( |
|||
// maxFloat40Value is the maximum value that the Float40 can have
|
|||
// (40 bits: maxFloat40Value=2**40-1)
|
|||
maxFloat40Value = 0xffffffffff |
|||
// Float40BytesLength defines the length of the Float40 values
|
|||
// represented as byte arrays
|
|||
Float40BytesLength = 5 |
|||
) |
|||
|
|||
var ( |
|||
// ErrFloat40Overflow is used when a given Float40 overflows the
|
|||
// maximum capacity of the Float40 (2**40-1)
|
|||
ErrFloat40Overflow = errors.New("Float40 overflow, max value: 2**40 -1") |
|||
// ErrFloat40E31 is used when the e > 31 when trying to convert a
|
|||
// *big.Int to Float40
|
|||
ErrFloat40E31 = errors.New("Float40 error, e > 31") |
|||
// ErrFloat40NotEnoughPrecission is used when the given *big.Int can
|
|||
// not be represented as Float40 due not enough precission
|
|||
ErrFloat40NotEnoughPrecission = errors.New("Float40 error, not enough precission") |
|||
) |
|||
|
|||
// Float40 represents a float in a 64 bit format
|
|||
type Float40 uint64 |
|||
|
|||
// Bytes return a byte array of length 5 with the Float40 value encoded in
|
|||
// BigEndian
|
|||
func (f40 Float40) Bytes() ([]byte, error) { |
|||
if f40 > maxFloat40Value { |
|||
return []byte{}, tracerr.Wrap(ErrFloat40Overflow) |
|||
} |
|||
|
|||
var f40Bytes [8]byte |
|||
binary.BigEndian.PutUint64(f40Bytes[:], uint64(f40)) |
|||
var b [5]byte |
|||
copy(b[:], f40Bytes[3:]) |
|||
return b[:], nil |
|||
} |
|||
|
|||
// Float40FromBytes returns a Float40 from a byte array of 5 bytes in Bigendian
|
|||
// representation.
|
|||
func Float40FromBytes(b []byte) Float40 { |
|||
var f40Bytes [8]byte |
|||
copy(f40Bytes[3:], b[:]) |
|||
f40 := binary.BigEndian.Uint64(f40Bytes[:]) |
|||
return Float40(f40) |
|||
} |
|||
|
|||
// BigInt converts the Float40 to a *big.Int v, where v = m * 10^e, being:
|
|||
// [ e | m ]
|
|||
// [ 5 bits | 35 bits ]
|
|||
func (f40 Float40) BigInt() (*big.Int, error) { |
|||
// take the 5 used bytes (FF * 5)
|
|||
var f40Uint64 uint64 = uint64(f40) & 0x00_00_00_FF_FF_FF_FF_FF |
|||
f40Bytes, err := f40.Bytes() |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
e := f40Bytes[0] & 0xF8 >> 3 // take first 5 bits
|
|||
m := f40Uint64 & 0x07_FF_FF_FF_FF // take the others 35 bits
|
|||
|
|||
exp := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(e)), nil) |
|||
r := new(big.Int).Mul(big.NewInt(int64(m)), exp) |
|||
return r, nil |
|||
} |
|||
|
|||
// NewFloat40 encodes a *big.Int integer as a Float40, returning error in case
|
|||
// of loss during the encoding.
|
|||
func NewFloat40(f *big.Int) (Float40, error) { |
|||
m := f |
|||
e := big.NewInt(0) |
|||
zero := big.NewInt(0) |
|||
ten := big.NewInt(10) |
|||
thres := big.NewInt(0x08_00_00_00_00) |
|||
for new(big.Int).Mod(m, ten).Cmp(zero) == 0 && m.Cmp(thres) >= 0 { |
|||
m = new(big.Int).Div(m, ten) |
|||
e = new(big.Int).Add(e, big.NewInt(1)) |
|||
} |
|||
if e.Int64() > 31 { |
|||
return 0, ErrFloat40E31 |
|||
} |
|||
if m.Cmp(thres) >= 0 { |
|||
return 0, ErrFloat40NotEnoughPrecission |
|||
} |
|||
r := new(big.Int).Add(m, |
|||
new(big.Int).Mul(e, thres)) |
|||
return Float40(r.Uint64()), nil |
|||
} |
|||
@ -0,0 +1,95 @@ |
|||
package common |
|||
|
|||
import ( |
|||
"math/big" |
|||
"testing" |
|||
|
|||
"github.com/stretchr/testify/assert" |
|||
"github.com/stretchr/testify/require" |
|||
) |
|||
|
|||
func TestConversionsFloat40(t *testing.T) { |
|||
testVector := map[Float40]string{ |
|||
6*0x800000000 + 123: "123000000", |
|||
2*0x800000000 + 4545: "454500", |
|||
30*0x800000000 + 10235: "10235000000000000000000000000000000", |
|||
0x000000000: "0", |
|||
0x800000000: "0", |
|||
0x0001: "1", |
|||
0x0401: "1025", |
|||
0x800000000 + 1: "10", |
|||
0xFFFFFFFFFF: "343597383670000000000000000000000000000000", |
|||
} |
|||
|
|||
for test := range testVector { |
|||
fix, err := test.BigInt() |
|||
require.NoError(t, err) |
|||
assert.Equal(t, fix.String(), testVector[test]) |
|||
|
|||
bi, ok := new(big.Int).SetString(testVector[test], 10) |
|||
require.True(t, ok) |
|||
|
|||
fl, err := NewFloat40(bi) |
|||
assert.NoError(t, err) |
|||
|
|||
fx2, err := fl.BigInt() |
|||
require.NoError(t, err) |
|||
assert.Equal(t, fx2.String(), testVector[test]) |
|||
} |
|||
} |
|||
|
|||
func TestExpectError(t *testing.T) { |
|||
testVector := map[string]error{ |
|||
"9922334455000000000000000000000000000000": nil, |
|||
"9922334455000000000000000000000000000001": ErrFloat40NotEnoughPrecission, |
|||
"9922334454999999999999999999999999999999": ErrFloat40NotEnoughPrecission, |
|||
"42949672950000000000000000000000000000000": nil, |
|||
"99223344556573838487575": ErrFloat40NotEnoughPrecission, |
|||
"992233445500000000000000000000000000000000": ErrFloat40E31, |
|||
"343597383670000000000000000000000000000000": nil, |
|||
"343597383680000000000000000000000000000000": ErrFloat40NotEnoughPrecission, |
|||
"343597383690000000000000000000000000000000": ErrFloat40NotEnoughPrecission, |
|||
"343597383700000000000000000000000000000000": ErrFloat40E31, |
|||
} |
|||
for test := range testVector { |
|||
bi, ok := new(big.Int).SetString(test, 10) |
|||
require.True(t, ok) |
|||
_, err := NewFloat40(bi) |
|||
assert.Equal(t, testVector[test], err) |
|||
} |
|||
} |
|||
|
|||
func BenchmarkFloat40(b *testing.B) { |
|||
newBigInt := func(s string) *big.Int { |
|||
bigInt, ok := new(big.Int).SetString(s, 10) |
|||
if !ok { |
|||
panic("Can not convert string to *big.Int") |
|||
} |
|||
return bigInt |
|||
} |
|||
type pair struct { |
|||
Float40 Float40 |
|||
BigInt *big.Int |
|||
} |
|||
testVector := []pair{ |
|||
{6*0x800000000 + 123, newBigInt("123000000")}, |
|||
{2*0x800000000 + 4545, newBigInt("454500")}, |
|||
{30*0x800000000 + 10235, newBigInt("10235000000000000000000000000000000")}, |
|||
{0x000000000, newBigInt("0")}, |
|||
{0x800000000, newBigInt("0")}, |
|||
{0x0001, newBigInt("1")}, |
|||
{0x0401, newBigInt("1025")}, |
|||
{0x800000000 + 1, newBigInt("10")}, |
|||
{0xFFFFFFFFFF, newBigInt("343597383670000000000000000000000000000000")}, |
|||
} |
|||
b.Run("NewFloat40()", func(b *testing.B) { |
|||
for i := 0; i < b.N; i++ { |
|||
_, _ = NewFloat40(testVector[i%len(testVector)].BigInt) |
|||
} |
|||
}) |
|||
b.Run("Float40.BigInt()", func(b *testing.B) { |
|||
for i := 0; i < b.N; i++ { |
|||
_, _ = testVector[i%len(testVector)].Float40.BigInt() |
|||
} |
|||
}) |
|||
} |
|||