From 8de728159707989d483a135eff9902ed5764687a Mon Sep 17 00:00:00 2001 From: arnaucube Date: Thu, 13 Dec 2018 22:12:35 +0100 Subject: [PATCH] merkletree --- .gitignore | 3 + README.md | 6 + db.go | 41 ++++ go.mod | 10 + go.sum | 99 +++++++++ merkletree.go | 352 ++++++++++++++++++++++++++++++++ merkletreeSpecification_test.go | 162 +++++++++++++++ merkletree_test.go | 351 +++++++++++++++++++++++++++++++ node.go | 36 ++++ print.go | 63 ++++++ utils.go | 61 ++++++ utils_test.go | 32 +++ 12 files changed, 1216 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 db.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 merkletree.go create mode 100644 merkletreeSpecification_test.go create mode 100644 merkletree_test.go create mode 100644 node.go create mode 100644 print.go create mode 100644 utils.go create mode 100644 utils_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f53c45d --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +leveldb +fmt +tmp diff --git a/README.md b/README.md new file mode 100644 index 0000000..e87a2f7 --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# go-merkletree [![Go Report Card](https://goreportcard.com/badge/github.com/arnaucube/go-merkletree)](https://goreportcard.com/report/github.com/arnaucube/go-merkletree) [![GoDoc](https://godoc.org/github.com/iden3/arnaucube/go-merkletree?status.svg)](https://godoc.org/github.com/iden3/arnaucube/go-merkletree) +Optimized MerkleTree implementation in Go. + +The MerkleTree is optimized in the design and concepts, to have a faster and lighter MerkleTree, maintaining compatibility with a non optimized MerkleTree. In this way, the MerkleRoot of the optimized MerkleTree will be the same that the MerkleRoot of the non optimized MerkleTree. + +This repo is holds the nostalgic (old) version of the MerkleTree implementation that we used in the past in iden3, as now has been substituted by a new specification. diff --git a/db.go b/db.go new file mode 100644 index 0000000..2d184ed --- /dev/null +++ b/db.go @@ -0,0 +1,41 @@ +package merkletree + +import ( + "bytes" + + "github.com/fatih/color" + common3 "github.com/iden3/go-iden3/common" +) + +func (mt *MerkleTree) Insert(key Hash, nodeType byte, indexLength uint32, nodeBytes []byte) error { + // add nodetype at the first byte of the value + var value []byte + value = append(value, nodeType) + indexLengthBytes := common3.Uint32ToBytes(indexLength) + value = append(value, indexLengthBytes[:]...) + value = append(value, nodeBytes[:]...) + + err := mt.storage.Put(key[:], value, nil) + if err != nil { + color.Red(err.Error()) + return err + } + return nil +} + +func (mt *MerkleTree) Get(key Hash) (byte, uint32, []byte, error) { + if bytes.Equal(key[:], EmptyNodeValue[:]) { + return 0, 0, EmptyNodeValue[:], nil + } + + value, err := mt.storage.Get(key[:], nil) + if err != nil { + return 0, 0, EmptyNodeValue[:], err + } + + // get nodetype of the first byte of the value + nodeType := value[0] + indexLength := common3.BytesToUint32(value[1:5]) + nodeBytes := value[5:] + return nodeType, indexLength, nodeBytes, err +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8ba2a48 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/arnaucube/merkletree + +require ( + github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 + github.com/ethereum/go-ethereum v1.8.20 + github.com/fatih/color v1.7.0 + github.com/iden3/go-iden3 v0.0.0-20181213124156-20b0f78e14ef + github.com/stretchr/testify v1.2.2 + github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..501c670 --- /dev/null +++ b/go.sum @@ -0,0 +1,99 @@ +github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= +github.com/aristanetworks/goarista v0.0.0-20180907105523-ff33da284e76/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= +github.com/arnaucube/cryptofun v0.0.0-20181124004321-9b11ae8280bd/go.mod h1:PZE8kKpHPD1UMrS3mTfAMmEEinGtijSwjxLRqRcD64A= +github.com/arnaucube/go-snark v0.0.0-20181212130559-7aafcfd5f3f4/go.mod h1:gLycS/B43DufBaH0jH8kqiE4A7w5FdOM8I9S416xh2Y= +github.com/btcsuite/btcd v0.0.0-20181013004428-67e573d211ac/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= +github.com/cespare/cp v1.0.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfDSMuaPjBr4cf6k7pwQQANm/yLKU= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= +github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/ethereum/go-ethereum v1.8.16/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY= +github.com/ethereum/go-ethereum v1.8.20 h1:Sr6DLbdc7Fl2IMDC0sjF2wO1jTO5nALFC1SoQnyAQEk= +github.com/ethereum/go-ethereum v1.8.20/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gin-contrib/cors v0.0.0-20180926132136-4f98e8b8e930/go.mod h1:cw+u9IsAkC16e42NtYYVCLsHYXE98nB3M7Dr9mLSeH4= +github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gxed/hashland v0.0.0-20180221191214-d9f6b97f8db2/go.mod h1:YUhWml1NaWLTNBl4NPptkB8MadfaIhgq+a2TRc+Mw4Q= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iden3/go-iden3 v0.0.0-20181213124156-20b0f78e14ef h1:HHgCgIGUb6TUOV6YnA/z9gxlPlzxdlgROWPrpEkkSMs= +github.com/iden3/go-iden3 v0.0.0-20181213124156-20b0f78e14ef/go.mod h1:kBO77X3PdJ9EglJy0h/0BKYTa4Hu8Wa+QRSbeXdUi8A= +github.com/ipfs/go-ipfs-api v1.3.5/go.mod h1:YWGjU+7Bdls1CpvsKsV6EsQ/KMyQqSpBru2hme/5WQg= +github.com/ipfs/go-ipfs-cmdkit v1.1.3/go.mod h1:9FtbMdUabcSqv/G4/8WCxSLxkZxn/aZEFrxxqnVcRbg= +github.com/ipfsconsortium/go-ipfsc v0.0.0-20180821102820-2014c66a1b4a/go.mod h1:4MbfcV8YX3CWjkWgUIH4vEVk002kMJlOEmhoSJP8SeE= +github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= +github.com/libp2p/go-libp2p-crypto v2.0.1+incompatible/go.mod h1:WHpT3tvhh7GM2INNJhQBuI6J+5z/o3QI0lTF5UVjppk= +github.com/libp2p/go-libp2p-metrics v2.1.7+incompatible/go.mod h1:ko4lRyuvbgwwxD2TJvt2RHONahjJlkn6l7L/iEbJBf0= +github.com/libp2p/go-libp2p-peer v2.4.0+incompatible/go.mod h1:fS2eFKRO1IomwBAf+SuE8P1XOT/AAiqSgVPNIFA7Jc0= +github.com/libp2p/go-libp2p-protocol v1.0.0/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= +github.com/libp2p/go-libp2p-pubsub v0.9.36/go.mod h1:E2KoEMwM5nWtdGV+wWueN7g/j++VL9tmCfjzk1fLpWc= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mgutz/logxi v0.0.0-20161027140823-aebf8a7d67ab/go.mod h1:y1pL58r5z2VvAjeG1VLGc8zOQgSOzbKN7kMHPvFXJ+8= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.0.0-20181005183134-51976451ce19/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/multiformats/go-multiaddr v1.3.0/go.mod h1:1JAWc2R8uiQTLrCHI/lmOkXYu5B8025fQbZjq8//YgY= +github.com/multiformats/go-multiaddr-net v1.6.3/go.mod h1:AO4WqKzxLt+paJ0N0kufj6teQ2R6fZbnItDvGTwilmk= +github.com/multiformats/go-multihash v1.0.8/go.mod h1:sT17phG+xVgnrZc8ht/ZoCIV0sKRwvmZkXk46UfSxM4= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= +github.com/rs/cors v1.5.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/sirupsen/logrus v1.1.0 h1:65VZabgUiV9ktjGM5nTq0+YurgTyX+YI2lSSfDjI+qU= +github.com/sirupsen/logrus v1.1.0/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.2.1/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc= +github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/ugorji/go/codec v0.0.0-20180927125128-99ea80c8b19a/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7 h1:bit1t3mgdR35yN0cX0G8orgLtOuyL9Wqxa1mccLB0ig= +golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/merkletree.go b/merkletree.go new file mode 100644 index 0000000..9985508 --- /dev/null +++ b/merkletree.go @@ -0,0 +1,352 @@ +package merkletree + +import ( + "bytes" + "errors" + + "github.com/syndtr/goleveldb/leveldb" +) + +const ( + // EmptyNodeType indicates the type of an EmptyNodeValue Node + EmptyNodeType = 00 + // NormalNodeType indicates the type of a middle Node + normalNodeType = 01 + // FinalNodeType indicates the type of middle Node that is in an optimized branch, then in the value contains the value of the final leaf node of that branch + finalNodeType = 02 + // ValueNodeType indicates the type of a value Node + valueNodeType = 03 + // RootNodeType indicates the type of a root Node + rootNodeType = 04 +) + +var ( + // ErrNodeAlreadyExists is an error that indicates that a node already exists in the merkletree database + ErrNodeAlreadyExists = errors.New("node already exists") + rootNodeValue = HashBytes([]byte("root")) + // EmptyNodeValue is a [32]byte EmptyNodeValue array, all to zero + EmptyNodeValue = Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} +) + +// Hash used in this tree, is the [32]byte keccak() +type Hash [32]byte + +// Value is the interface of a generic leaf, a key value object stored in the leveldb +type Value interface { + IndexLength() uint32 // returns the index length value + Bytes() []byte // returns the value in byte array representation +} + +//MerkleTree struct with the main elements of the Merkle Tree +type MerkleTree struct { + // sync.RWMutex + storage *leveldb.DB + root Hash + numLevels int // Height of the Merkle Tree, number of levels +} + +// New generates a new Merkle Tree +func New(storage *leveldb.DB, numLevels int) (*MerkleTree, error) { + var mt MerkleTree + mt.storage = storage + mt.numLevels = numLevels + var err error + _, _, rootHash, err := mt.Get(rootNodeValue) + if err != nil { + mt.root = EmptyNodeValue + err = mt.Insert(rootNodeValue, rootNodeType, 0, mt.root[:]) + if err != nil { + return nil, err + } + } + copy(mt.root[:], rootHash) + return &mt, nil +} + +// Root returns the merkletree.Root +func (mt *MerkleTree) Root() Hash { + return mt.root +} + +// NumLevels returns the merkletree.NumLevels +func (mt *MerkleTree) NumLevels() int { + return mt.numLevels +} + +// Add adds the leaf to the MT +func (mt *MerkleTree) Add(v Value) error { + // add the leaf that we are adding + mt.Insert(HashBytes(v.Bytes()), valueNodeType, v.IndexLength(), v.Bytes()) + + hi := HashBytes(v.Bytes()[:v.IndexLength()]) + path := getPath(mt.numLevels, hi) + + nodeHash := mt.root + var siblings []Hash + for i := mt.numLevels - 2; i >= 0; i-- { + nodeType, indexLength, nodeBytes, err := mt.Get(nodeHash) + if err != nil { + return err + } + if nodeType == byte(finalNodeType) { + hiChild := HashBytes(nodeBytes[:indexLength]) + pathChild := getPath(mt.numLevels, hiChild) + posDiff := comparePaths(pathChild, path) + if posDiff == -1 { + return ErrNodeAlreadyExists + } + finalNode1Hash := calcHashFromLeafAndLevel(posDiff, pathChild, HashBytes(nodeBytes)) + mt.Insert(finalNode1Hash, finalNodeType, indexLength, nodeBytes) + finalNode2Hash := calcHashFromLeafAndLevel(posDiff, path, HashBytes(v.Bytes())) + mt.Insert(finalNode2Hash, finalNodeType, v.IndexLength(), v.Bytes()) + // now the parent + var parentNode treeNode + if path[posDiff] { + parentNode = treeNode{ + ChildL: finalNode1Hash, + ChildR: finalNode2Hash, + } + } else { + parentNode = treeNode{ + ChildL: finalNode2Hash, + ChildR: finalNode1Hash, + } + } + siblings = append(siblings, getEmptiesBetweenIAndPosHash(mt, i, posDiff+1)...) + if mt.root, err = mt.replaceLeaf(siblings, path[posDiff+1:], parentNode.Ht(), normalNodeType, 0, parentNode.Bytes()); err != nil { + return err + } + mt.Insert(rootNodeValue, rootNodeType, 0, mt.root[:]) + return nil + } + node := parseNodeBytes(nodeBytes) + var sibling Hash + if !path[i] { + nodeHash = node.ChildL + sibling = node.ChildR + } else { + nodeHash = node.ChildR + sibling = node.ChildL + } + siblings = append(siblings, sibling) + + if bytes.Equal(nodeHash[:], EmptyNodeValue[:]) { + // if the node is EmptyNodeValue, the leaf data will go directly at that height, as a Final Node + if i == mt.numLevels-2 && bytes.Equal(siblings[len(siblings)-1][:], EmptyNodeValue[:]) { + // if the pt node is the unique in the tree, just put it into the root node + // this means to be in i==mt.NumLevels-2 && nodeHash==EmptyNodeValue + finalNodeHash := calcHashFromLeafAndLevel(i+1, path, HashBytes(v.Bytes())) + mt.Insert(finalNodeHash, finalNodeType, v.IndexLength(), v.Bytes()) + mt.root = finalNodeHash + mt.Insert(rootNodeValue, rootNodeType, 0, mt.root[:]) + return nil + } + finalNodeHash := calcHashFromLeafAndLevel(i, path, HashBytes(v.Bytes())) + if mt.root, err = mt.replaceLeaf(siblings, path[i:], finalNodeHash, finalNodeType, v.IndexLength(), v.Bytes()); err != nil { + return err + } + mt.Insert(rootNodeValue, rootNodeType, 0, mt.root[:]) + return nil + } + } + + var err error + mt.root, err = mt.replaceLeaf(siblings, path, HashBytes(v.Bytes()), valueNodeType, v.IndexLength(), v.Bytes()) + if err != nil { + return err + } + mt.Insert(rootNodeValue, rootNodeType, 0, mt.root[:]) + return nil +} + +// GenerateProof generates the Merkle Proof from a given leafHash for the current root +func (mt *MerkleTree) GenerateProof(hi Hash) ([]byte, error) { + var empties [32]byte + + path := getPath(mt.numLevels, hi) + var siblings []Hash + nodeHash := mt.root + + for level := 0; level < mt.numLevels-1; level++ { + nodeType, indexLength, nodeBytes, err := mt.Get(nodeHash) + if err != nil { + return nil, err + } + if nodeType == byte(finalNodeType) { + realValueInPos, err := mt.GetValueInPos(hi) + if err != nil { + return nil, err + } + if bytes.Equal(realValueInPos[:], EmptyNodeValue[:]) { + // go until the path is different, then get the nodes between this FinalNode and the node in the diffPath, they will be the siblings of the merkle proof + leafHi := HashBytes(nodeBytes[:indexLength]) // hi of element that was in the end of the branch (the finalNode) + pathChild := getPath(mt.numLevels, leafHi) + + // get the position where the path is different + posDiff := comparePaths(pathChild, path) + if posDiff == -1 { + return nil, ErrNodeAlreadyExists + } + + if posDiff != mt.NumLevels()-1-level { + sibling := calcHashFromLeafAndLevel(posDiff, pathChild, HashBytes(nodeBytes)) + setbitmap(empties[:], uint(mt.NumLevels()-2-posDiff)) + siblings = append([]Hash{sibling}, siblings...) + } + + } + break + } + node := parseNodeBytes(nodeBytes) + + var sibling Hash + if !path[mt.numLevels-level-2] { + nodeHash = node.ChildL + sibling = node.ChildR + } else { + nodeHash = node.ChildR + sibling = node.ChildL + } + if !bytes.Equal(sibling[:], EmptyNodeValue[:]) { + setbitmap(empties[:], uint(level)) + siblings = append([]Hash{sibling}, siblings...) + } + } + // merge empties and siblings + var mp []byte + mp = append(mp, empties[:]...) + for k := range siblings { + mp = append(mp, siblings[k][:]...) + } + return mp, nil +} + +// GetValueInPos returns the merkletree value in the position of the Hash of the Index (Hi) +func (mt *MerkleTree) GetValueInPos(hi Hash) ([]byte, error) { + path := getPath(mt.numLevels, hi) + nodeHash := mt.root + for i := mt.numLevels - 2; i >= 0; i-- { + nodeType, indexLength, nodeBytes, err := mt.Get(nodeHash) + if err != nil { + return nodeBytes, err + } + if nodeType == byte(finalNodeType) { + // check if nodeBytes path is different of hi + index := nodeBytes[:indexLength] + hi := HashBytes(index) + nodePath := getPath(mt.numLevels, hi) + posDiff := comparePaths(path, nodePath) + // if is different, return an EmptyNodeValue, else return the nodeBytes + if posDiff != -1 { + return EmptyNodeValue[:], nil + } + return nodeBytes, nil + } + node := parseNodeBytes(nodeBytes) + if !path[i] { + nodeHash = node.ChildL + } else { + nodeHash = node.ChildR + } + } + _, _, valueBytes, err := mt.Get(nodeHash) + if err != nil { + return valueBytes, err + } + return valueBytes, nil +} + +func calcHashFromLeafAndLevel(untilLevel int, path []bool, leafHash Hash) Hash { + nodeCurrLevel := leafHash + for i := 0; i < untilLevel; i++ { + if path[i] { + node := treeNode{ + ChildL: EmptyNodeValue, + ChildR: nodeCurrLevel, + } + nodeCurrLevel = node.Ht() + } else { + node := treeNode{ + ChildL: nodeCurrLevel, + ChildR: EmptyNodeValue, + } + nodeCurrLevel = node.Ht() + } + } + return nodeCurrLevel +} + +func (mt *MerkleTree) replaceLeaf(siblings []Hash, path []bool, newLeafHash Hash, nodetype byte, indexLength uint32, newLeafValue []byte) (Hash, error) { + // add the new leaf + mt.Insert(newLeafHash, nodetype, indexLength, newLeafValue) + currNode := newLeafHash + // here the path is only the path[posDiff+1] + for i := 0; i < len(siblings); i++ { + if !path[i] { + node := treeNode{ + ChildL: currNode, + ChildR: siblings[len(siblings)-1-i], + } + mt.Insert(node.Ht(), normalNodeType, 0, node.Bytes()) + currNode = node.Ht() + } else { + + node := treeNode{ + ChildL: siblings[len(siblings)-1-i], + ChildR: currNode, + } + mt.Insert(node.Ht(), normalNodeType, 0, node.Bytes()) + currNode = node.Ht() + } + } + + return currNode, nil // currNode = root +} + +// CheckProof validates the Merkle Proof for the leafHash and root +func CheckProof(root Hash, proof []byte, hi Hash, ht Hash, numLevels int) bool { + var empties [32]byte + copy(empties[:], proof[:len(empties)]) + hashLen := len(EmptyNodeValue) + + var siblings []Hash + for i := len(empties); i < len(proof); i += hashLen { + var siblingHash Hash + copy(siblingHash[:], proof[i:i+hashLen]) + siblings = append(siblings, siblingHash) + } + + path := getPath(numLevels, hi) + nodeHash := ht + siblingUsedPos := 0 + + for level := numLevels - 2; level >= 0; level-- { + var sibling Hash + if testbitmap(empties[:], uint(level)) { + sibling = siblings[siblingUsedPos] + siblingUsedPos++ + } else { + sibling = EmptyNodeValue + } + // calculate the nodeHash with the current nodeHash and the sibling + var node treeNode + if path[numLevels-level-2] { + node = treeNode{ + ChildL: sibling, + ChildR: nodeHash, + } + } else { + node = treeNode{ + ChildL: nodeHash, + ChildR: sibling, + } + } + // if both childs are EmptyNodeValue, the parent will be EmptyNodeValue + if bytes.Equal(nodeHash[:], EmptyNodeValue[:]) && bytes.Equal(sibling[:], EmptyNodeValue[:]) { + nodeHash = EmptyNodeValue + } else { + nodeHash = node.Ht() + } + } + return bytes.Equal(nodeHash[:], root[:]) +} diff --git a/merkletreeSpecification_test.go b/merkletreeSpecification_test.go new file mode 100644 index 0000000..45219af --- /dev/null +++ b/merkletreeSpecification_test.go @@ -0,0 +1,162 @@ +package merkletree + +/* +This is just an example of basic tests for the iden3-merkletree-specification. +The methods and variables names can be different. +*/ + +import ( + "strconv" + "testing" + + common3 "github.com/iden3/go-iden3/common" + "github.com/stretchr/testify/assert" +) + +type testBytesLeaf struct { + data []byte + indexLength uint32 +} + +func (c testBytesLeaf) Bytes() (b []byte) { + return c.data +} +func (c testBytesLeaf) IndexLength() uint32 { + return c.indexLength +} +func (c testBytesLeaf) Hi() Hash { + h := HashBytes(c.Bytes()[:c.IndexLength()]) + return h +} +func newTestBytesLeaf(data string, indexLength uint32) testBytesLeaf { + return testBytesLeaf{ + data: []byte(data), + indexLength: indexLength, + } +} + +// Test to check the iden3-merkletree-specification +func TestIden3MerkletreeSpecification(t *testing.T) { + h := HashBytes([]byte("test")).Hex() + assert.Equal(t, "0x9c22ff5f21f0b81b113e63f7db6da94fedef11b2119b4088b89664fb9a3cb658", h) + + h = HashBytes([]byte("authorizeksign")).Hex() + assert.Equal(t, "0x353f867ef725411de05e3d4b0a01c37cf7ad24bcc213141a05ed7726d7932a1f", h) + + mt := newTestingMerkle(t, 140) + defer mt.storage.Close() + + // empty tree + assert.Equal(t, "0x0000000000000000000000000000000000000000000000000000000000000000", mt.Root().Hex()) + + // add leaf + leaf := testBytesLeaf{ + data: []byte("this is a test leaf"), + indexLength: 15, + } + assert.Nil(t, mt.Add(leaf)) + assert.Equal(t, "0xb4fdf8a653198f0e179ccb3af7e4fc09d76247f479d6cfc95cd92d6fda589f27", mt.Root().Hex()) + + // proof with only one leaf in the MerkleTree + proof, err := mt.GenerateProof(leaf.Hi()) + assert.Nil(t, err) + assert.Equal(t, "0x0000000000000000000000000000000000000000000000000000000000000000", common3.BytesToHex(proof)) + + // add a second leaf + leaf2 := testBytesLeaf{ + data: []byte("this is a second test leaf"), + indexLength: 15, + } + err = mt.Add(leaf2) + assert.Nil(t, err) + assert.Equal(t, "0x8ac95e9c8a6fbd40bb21de7895ee35f9c8f30ca029dbb0972c02344f49462e82", mt.Root().Hex()) + + // proof of the second leaf, with two leafs in the MerkleTree + proof2, err := mt.GenerateProof(leaf2.Hi()) + assert.Nil(t, err) + assert.Equal(t, "0x0000000000000000000000000000000000000000000000000000000000000001fd8e1a60cdb23c0c7b2cf8462c99fafd905054dccb0ed75e7c8a7d6806749b6b", common3.BytesToHex(proof2)) + + // proof of emptyLeaf + leaf3 := testBytesLeaf{ + data: []byte("this is a third test leaf"), + indexLength: 15, + } + proof3, err := mt.GenerateProof(leaf3.Hi()) + assert.Nil(t, err) + assert.Equal(t, "0x000000000000000000000000000000000000000000000000000000000000000389741fa23da77c259781ad8f4331a5a7d793eef1db7e5200ddfc8e5f5ca7ce2bfd8e1a60cdb23c0c7b2cf8462c99fafd905054dccb0ed75e7c8a7d6806749b6b", common3.BytesToHex(proof3)) + + // getLeafByHi/GetValueInPos + bytesInHi, err := mt.GetValueInPos(leaf2.Hi()) + assert.Nil(t, err) + assert.Equal(t, leaf2.Bytes(), bytesInHi) + + // check proof + rootBytes, err := common3.HexToBytes("0x7d7c5e8f4b3bf434f3d9d223359c4415e2764dd38de2e025fbf986e976a7ed3d") + assert.Nil(t, err) + mp, err := common3.HexToBytes("0x0000000000000000000000000000000000000000000000000000000000000002d45aada6eec346222eaa6b5d3a9260e08c9b62fcf63c72bc05df284de07e6a52") + assert.Nil(t, err) + hiBytes, err := common3.HexToBytes("0x786677808ba77bdd9090a969f1ef2cbd1ac5aecd9e654f340500159219106878") + assert.Nil(t, err) + htBytes, err := common3.HexToBytes("0x786677808ba77bdd9090a969f1ef2cbd1ac5aecd9e654f340500159219106878") + assert.Nil(t, err) + var root, hi, ht Hash + copy(root[:], rootBytes) + copy(hi[:], hiBytes) + copy(ht[:], htBytes) + verified := CheckProof(root, mp, hi, ht, 140) + assert.True(t, verified) + + // check proof of empty + rootBytes, err = common3.HexToBytes("0x8f021d00c39dcd768974ddfe0d21f5d13f7215bea28db1f1cb29842b111332e7") + assert.Nil(t, err) + mp, err = common3.HexToBytes("0x0000000000000000000000000000000000000000000000000000000000000004bf8e980d2ed328ae97f65c30c25520aeb53ff837579e392ea1464934c7c1feb9") + assert.Nil(t, err) + hiBytes, err = common3.HexToBytes("0xa69792a4cff51f40b7a1f7ae596c6ded4aba241646a47538898f17f2a8dff647") + assert.Nil(t, err) + htBytes, err = common3.HexToBytes("0x0000000000000000000000000000000000000000000000000000000000000000") + assert.Nil(t, err) + copy(root[:], rootBytes) + copy(hi[:], hiBytes) + copy(ht[:], htBytes) + verified = CheckProof(root, mp, hi, ht, 140) + assert.True(t, verified) + + // check the proof generated in the previous steps + verified = CheckProof(mt.Root(), proof2, leaf2.Hi(), HashBytes(leaf2.Bytes()), 140) + assert.True(t, verified) + // check proof of no existence (emptyLeaf), as we are prooving an empty leaf, the Ht is an empty value (0x000...0) + verified = CheckProof(mt.Root(), proof3, leaf3.Hi(), EmptyNodeValue, 140) + assert.True(t, verified) + + // add leafs in different orders + mt1 := newTestingMerkle(t, 140) + defer mt.storage.Close() + + mt1.Add(newTestBytesLeaf("0 this is a test leaf", 15)) + mt1.Add(newTestBytesLeaf("1 this is a test leaf", 15)) + mt1.Add(newTestBytesLeaf("2 this is a test leaf", 15)) + mt1.Add(newTestBytesLeaf("3 this is a test leaf", 15)) + mt1.Add(newTestBytesLeaf("4 this is a test leaf", 15)) + + mt2 := newTestingMerkle(t, 140) + defer mt.storage.Close() + + mt2.Add(newTestBytesLeaf("2 this is a test leaf", 15)) + mt2.Add(newTestBytesLeaf("1 this is a test leaf", 15)) + mt2.Add(newTestBytesLeaf("0 this is a test leaf", 15)) + mt2.Add(newTestBytesLeaf("3 this is a test leaf", 15)) + mt2.Add(newTestBytesLeaf("4 this is a test leaf", 15)) + + assert.Equal(t, mt1.Root().Hex(), mt2.Root().Hex()) + + // adding 1000 leafs + mt1000 := newTestingMerkle(t, 140) + defer mt.storage.Close() + + numToAdd := 1000 + for i := 0; i < numToAdd; i++ { + leaf := newTestBytesLeaf(strconv.Itoa(i)+" this is a test leaf", 15) + mt1000.Add(leaf) + } + assert.Equal(t, "0x6e2da580b2920cd78ed8d4e4bf41e209dfc99ef28bc19560042f0ac803e0d6f7", mt1000.Root().Hex()) +} diff --git a/merkletree_test.go b/merkletree_test.go new file mode 100644 index 0000000..a8d7133 --- /dev/null +++ b/merkletree_test.go @@ -0,0 +1,351 @@ +package merkletree + +import ( + "encoding/hex" + "fmt" + "strconv" + "testing" + "time" + + "github.com/dchest/uniuri" + common3 "github.com/iden3/go-iden3/common" + "github.com/stretchr/testify/assert" + "github.com/syndtr/goleveldb/leveldb" +) + +type testBase struct { + Length [4]byte + Namespace Hash + Type Hash + Version uint32 +} +type testLeaf struct { + testBase + extraIndex struct { + Data []byte + } +} + +func parseTestLeafBytes(b []byte) testLeaf { + var c testLeaf + copy(c.testBase.Length[:], b[0:4]) + copy(c.testBase.Namespace[:], b[4:36]) + copy(c.testBase.Type[:], b[36:68]) + versionBytes := b[68:72] + c.testBase.Version = common3.BytesToUint32(versionBytes) + c.extraIndex.Data = b[72:] + return c +} +func (c testLeaf) Bytes() (b []byte) { + b = append(b, c.testBase.Length[:]...) + b = append(b, c.testBase.Namespace[:]...) + b = append(b, c.testBase.Type[:]...) + versionBytes := common3.Uint32ToBytes(c.testBase.Version) + b = append(b, versionBytes[:]...) + b = append(b, c.extraIndex.Data[:]...) + return b +} +func (c testLeaf) IndexLength() uint32 { + return uint32(len(c.Bytes())) +} +func (c testLeaf) hi() Hash { + h := HashBytes(c.Bytes()) + return h +} +func newTestLeaf(namespaceStr, typeStr string, data []byte) testLeaf { + var c testLeaf + c.testBase.Length = [4]byte{0x00, 0x00, 0x00, 0x48} + c.testBase.Namespace = HashBytes([]byte(namespaceStr)) + c.testBase.Type = HashBytes([]byte(typeStr)) + c.testBase.Version = 0 + c.extraIndex.Data = data + return c +} + +type Fatalable interface { + Fatal(args ...interface{}) +} + +func newTestingMerkle(f Fatalable, numLevels int) *MerkleTree { + tim := time.Now().Unix() + tstr := fmt.Sprint(tim) + s := uniuri.New() + db, err := leveldb.OpenFile("tmp/db"+s+"-"+tstr, nil) + if err != nil { + f.Fatal(err) + return nil + } + // defer db.Close() + mt, err := New(db, numLevels) + if err != nil { + f.Fatal(err) + return nil + } + return mt +} + +func TestNewMT(t *testing.T) { + + //create a new MT + mt := newTestingMerkle(t, 140) + defer mt.storage.Close() + assert.Equal(t, "0x0000000000000000000000000000000000000000000000000000000000000000", mt.Root().Hex()) +} + +func TestAddLeaf(t *testing.T) { + + mt := newTestingMerkle(t, 140) + defer mt.storage.Close() + + leaf := newTestLeaf("iden3.io", "typespec", []byte("c1")) + assert.Equal(t, "0x939862c94ca9772fc9e2621df47128b1d4041b514e19edc969a92d8f0dae558f", leaf.hi().Hex()) + + assert.Nil(t, mt.Add(leaf)) + assert.Equal(t, "0x9d3c407ff02c813cd474c0a6366b4f7c58bf417a38268f7a0d73a8bca2490b9b", mt.Root().Hex()) +} + +func TestAddLeafs(t *testing.T) { + + mt := newTestingMerkle(t, 140) + defer mt.storage.Close() + + leaf := newTestLeaf("iden3.io", "typespec", []byte("c1")) + assert.Equal(t, "0x939862c94ca9772fc9e2621df47128b1d4041b514e19edc969a92d8f0dae558f", leaf.hi().Hex()) + assert.Nil(t, mt.Add(leaf)) + + assert.Nil(t, mt.Add(newTestLeaf("iden3.io2", "typespec2", []byte("c2")))) + assert.Equal(t, "0xebae8fb483b48ba6c337136535198eb8bcf891daba40ac81e28958c09b9b229b", mt.Root().Hex()) + + mt.Add(newTestLeaf("iden3.io3", "typespec3", []byte("c3"))) + mt.Add(newTestLeaf("iden3.io4", "typespec4", []byte("c4"))) + assert.Equal(t, "0xb4b51aa0c77a8e5ed0a099d7c11c7d2a9219ef241da84f0689da1f40a5f6ac31", mt.Root().Hex()) +} + +func TestAddLeafsCollision(t *testing.T) { + + mt := newTestingMerkle(t, 140) + defer mt.storage.Close() + + leaf := newTestLeaf("iden3.io", "typespec", []byte("c1")) + assert.Nil(t, mt.Add(leaf)) + + root1 := mt.Root() + assert.EqualError(t, mt.Add(leaf), ErrNodeAlreadyExists.Error()) + + assert.Equal(t, root1.Hex(), mt.Root().Hex()) +} + +func TestAddLeafsDifferentOrders(t *testing.T) { + + mt1 := newTestingMerkle(t, 140) + defer mt1.storage.Close() + + mt1.Add(newTestLeaf("iden3.io", "typespec", []byte("c1"))) + mt1.Add(newTestLeaf("iden3.io2", "typespec2", []byte("c2"))) + mt1.Add(newTestLeaf("iden3.io3", "typespec3", []byte("c3"))) + mt1.Add(newTestLeaf("iden3.io4", "typespec4", []byte("c4"))) + mt1.Add(newTestLeaf("iden3.io5", "typespec5", []byte("c5"))) + + mt2 := newTestingMerkle(t, 140) + defer mt2.storage.Close() + + mt2.Add(newTestLeaf("iden3.io3", "typespec3", []byte("c3"))) + mt2.Add(newTestLeaf("iden3.io2", "typespec2", []byte("c2"))) + mt2.Add(newTestLeaf("iden3.io", "typespec", []byte("c1"))) + mt2.Add(newTestLeaf("iden3.io4", "typespec4", []byte("c4"))) + mt2.Add(newTestLeaf("iden3.io5", "typespec5", []byte("c5"))) + + assert.Equal(t, mt1.Root().Hex(), mt2.Root().Hex()) +} +func TestBenchmarkAddingLeafs(t *testing.T) { + + mt := newTestingMerkle(t, 140) + defer mt.storage.Close() + + start := time.Now() + numToAdd := 1000 + for i := 0; i < numToAdd; i++ { + leaf := newTestLeaf("iden3.io"+strconv.Itoa(i), "typespec"+strconv.Itoa(i), []byte("c"+strconv.Itoa(i))) + mt.Add(leaf) + } + fmt.Print("time elapsed adding " + strconv.Itoa(numToAdd) + " leafs: ") + fmt.Println(time.Since(start)) +} +func BenchmarkAddingLeafs(b *testing.B) { + + mt := newTestingMerkle(b, 140) + defer mt.storage.Close() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + leaf := newTestLeaf("iden3.io"+strconv.Itoa(i), "typespec"+strconv.Itoa(i), []byte("c"+strconv.Itoa(i))) + if err := mt.Add(leaf); err != nil { + b.Fatal(err) + } + } +} + +func TestGenerateProof(t *testing.T) { + mt := newTestingMerkle(t, 140) + defer mt.storage.Close() + + mt.Add(newTestLeaf("iden3.io_3", "typespec_3", []byte("c3"))) + mt.Add(newTestLeaf("iden3.io_2", "typespec_2", []byte("c2"))) + + leaf1 := newTestLeaf("iden3.io_1", "typespec_1", []byte("c1")) + assert.Nil(t, mt.Add(leaf1)) + + mp, err := mt.GenerateProof(parseTestLeafBytes(leaf1.Bytes()).hi()) + assert.Nil(t, err) + + mpHexExpected := "0000000000000000000000000000000000000000000000000000000000000002beb0fd6dcf18d37fe51cf34beacd4c524d9c039ef9da2a27ccd3e7edf662c39c" + assert.Equal(t, mpHexExpected, hex.EncodeToString(mp)) +} + +func TestCheckProof(t *testing.T) { + mt := newTestingMerkle(t, 140) + defer mt.storage.Close() + + leaf1 := newTestLeaf("iden3.io_1", "typespec_1", []byte("c1")) + assert.Nil(t, mt.Add(leaf1)) + + leaf3 := newTestLeaf("iden3.io_3", "typespec_3", []byte("c3")) + assert.Nil(t, mt.Add(leaf3)) + + mp, err := mt.GenerateProof(parseTestLeafBytes(leaf1.Bytes()).hi()) + assert.Nil(t, err) + verified := CheckProof(mt.Root(), mp, leaf1.hi(), HashBytes(leaf1.Bytes()), mt.NumLevels()) + assert.True(t, verified) + +} + +func TestProofOfEmpty(t *testing.T) { // proof of a non revocated leaf, prove that is empty the hi position of the leaf.version+1 + mt := newTestingMerkle(t, 140) + defer mt.storage.Close() + + leaf1 := newTestLeaf("iden3.io_1", "typespec_1", []byte("c1")) + // proof when there is nothing in the tree + mp, err := mt.GenerateProof(leaf1.hi()) + assert.Nil(t, err) + verified := CheckProof(mt.Root(), mp, leaf1.hi(), EmptyNodeValue, mt.NumLevels()) + assert.True(t, verified) + + // add the first leaf + assert.Nil(t, mt.Add(leaf1)) + + // proof when there is only one leaf in the tree + leaf2 := newTestLeaf("iden3.io_2", "typespec_2", []byte("c2")) + mp, err = mt.GenerateProof(leaf2.hi()) + assert.Nil(t, err) + verified = CheckProof(mt.Root(), mp, leaf2.hi(), EmptyNodeValue, mt.NumLevels()) + assert.True(t, verified) + + // check that the value in Hi is Empty + valueInPos, err := mt.GetValueInPos(leaf2.hi()) + assert.Nil(t, err) + assert.Equal(t, EmptyNodeValue.Bytes(), valueInPos) +} + +func DifferentNonExistenceProofs(t *testing.T) { + mt1 := newTestingMerkle(t, 140) + defer mt1.storage.Close() + + mt2 := newTestingMerkle(t, 140) + defer mt2.storage.Close() + + leaf1 := newTestLeaf("iden3.io_1", "typespec_1", []byte("c1")) + leaf2 := newTestLeaf("iden3.io_1", "typespec_1", []byte("c2")) + + assert.Nil(t, mt1.Add(leaf1)) + assert.Nil(t, mt2.Add(leaf2)) + + leaf1.Version++ + leaf2.Version++ + + np1, err := mt1.GenerateProof(leaf1.hi()) + assert.Nil(t, err) + np2, err := mt2.GenerateProof(leaf2.hi()) + assert.Nil(t, err) + + assert.True(t, CheckProof(mt1.Root(), np1, leaf1.hi(), EmptyNodeValue, mt1.NumLevels())) + assert.True(t, CheckProof(mt2.Root(), np2, leaf2.hi(), EmptyNodeValue, mt2.NumLevels())) + + assert.Equal(t, "0000000000000000000000000000000000000000000000000000000000000010a40617c8c3390736831d00b2003e2133353190f5d3b3a586cf829f0f2009aacc", hex.EncodeToString(np1)) + assert.Equal(t, "0000000000000000000000000000000000000000000000000000000000000001b274a34a3bd95915fe982a0163e3e0a2f79a371b8307661341f8914e22b313e1", hex.EncodeToString(np2)) + +} + +func TestGetLeafInPos(t *testing.T) { + mt := newTestingMerkle(t, 140) + defer mt.storage.Close() + + for i := 0; i < 50; i++ { + leaf := newTestLeaf("iden3.io"+strconv.Itoa(i), "typespec"+strconv.Itoa(i), []byte("c"+strconv.Itoa(i))) + mt.Add(leaf) + + } + leaf1 := newTestLeaf("iden3.io_x", "typespec_x", []byte("cx")) + assert.Nil(t, mt.Add(leaf1)) + + leaf := parseTestLeafBytes(leaf1.Bytes()) + leafInPosBytes, err := mt.GetValueInPos(leaf.hi()) + assert.Nil(t, err) + assert.Equal(t, leaf1.Bytes(), leafInPosBytes) + + // empty value in position + leaf2 := newTestLeaf("iden3.io_y", "typespec_y", []byte("cy")) + leafInPosBytes, err = mt.GetValueInPos(leaf2.hi()) + assert.Nil(t, err) + assert.Equal(t, EmptyNodeValue[:], leafInPosBytes) +} + +type vt struct { + v []byte + idxlen uint32 +} + +func (v vt) IndexLength() uint32 { + return v.idxlen +} +func (v vt) Bytes() []byte { + return v.v +} + +func TestVector4(t *testing.T) { + mt := newTestingMerkle(t, 4) + defer mt.storage.Close() + + zeros := make([]byte, 32, 32) + zeros[31] = 1 // to avoid adding Empty element + assert.Nil(t, mt.Add(vt{zeros, uint32(1)})) + v := vt{zeros, uint32(2)} + assert.Nil(t, mt.Add(v)) + proof, _ := mt.GenerateProof(HashBytes(v.Bytes()[:v.IndexLength()])) + assert.True(t, CheckProof(mt.Root(), proof, HashBytes(v.Bytes()[:v.IndexLength()]), HashBytes(v.Bytes()), mt.NumLevels())) + assert.Equal(t, 4, mt.NumLevels()) + assert.Equal(t, "0000000000000000000000000000000000000000000000000000000000000001", hex.EncodeToString(v.Bytes())) + assert.Equal(t, "0xc1b95ffbb999a6dd7a472a610a98891ffae95cc973d1d1e21acfdd68db830b51", mt.Root().Hex()) + assert.Equal(t, "00000000000000000000000000000000000000000000000000000000000000023cf025e4b4fc3ebe57374bf0e0c78ceb0009bdc4466a45174d80e8f508d1a4e3", hex.EncodeToString(proof)) +} + +func TestVector140(t *testing.T) { + mt := newTestingMerkle(t, 140) + defer mt.storage.Close() + + zeros := make([]byte, 32, 32) + zeros[31] = 1 // to avoid adding Empty element + for i := 1; i < len(zeros)-1; i++ { + v := vt{zeros, uint32(i)} + assert.Nil(t, mt.Add(v)) + proof, err := mt.GenerateProof(HashBytes(v.Bytes()[:v.IndexLength()])) + assert.Nil(t, err) + assert.True(t, CheckProof(mt.Root(), proof, HashBytes(v.Bytes()[:v.IndexLength()]), HashBytes(v.Bytes()), mt.NumLevels())) + if i == len(zeros)-2 { + assert.Equal(t, 140, mt.NumLevels()) + assert.Equal(t, uint32(30), v.IndexLength()) + assert.Equal(t, "0000000000000000000000000000000000000000000000000000000000000001", hex.EncodeToString(v.Bytes())) + assert.Equal(t, "0x35f83288adf03bfb61d8d57fab9ed092da79833b58bbdbe9579b636753494ebd", mt.Root().Hex()) + assert.Equal(t, "000000000000000000000000000000000000000000000000000000000000001f0d1f363115f3333197a009b6674f46bba791308af220ad71515567702b3b44a2b540c1abad0ff81386a78b77e8907a56b7268d24513928ae83497adf4ad93a55e380267ead8305202da0640c1518e144dee87717c732b738fa182c6ef458defd6baf50022b01e3222715d4fca4c198e94536101f6ac314b3d261d3aaa0684395c1db60626e01c39fe4f69418055c2ebd70e0c07b6d9db5c4aed0a11ed2b6a773", hex.EncodeToString(proof)) + } + } +} diff --git a/node.go b/node.go new file mode 100644 index 0000000..b7c2987 --- /dev/null +++ b/node.go @@ -0,0 +1,36 @@ +package merkletree + +import "bytes" + +// treeNode is the data structure of an intermediate node of the Merkle Tree +type treeNode struct { + ChildL Hash // hash of the left child + ChildR Hash // hash of the right child +} + +// Bytes returns an array of bytes with the Node data +func (n *treeNode) Bytes() (b []byte) { + b = append(b, n.ChildL[:]...) + b = append(b, n.ChildR[:]...) + return b +} + +// Ht returns the hash of the full node +func (n *treeNode) Ht() Hash { + h := HashBytes(n.Bytes()) + return h +} + +// ParseNodeBytes returns a Node struct from an array of bytes +func parseNodeBytes(b []byte) treeNode { + if bytes.Equal(b, EmptyNodeValue[:]) { + var node treeNode + node.ChildL = EmptyNodeValue + node.ChildR = EmptyNodeValue + return node + } + var node treeNode + copy(node.ChildL[:], b[:32]) + copy(node.ChildR[:], b[32:]) + return node +} diff --git a/print.go b/print.go new file mode 100644 index 0000000..5bb8882 --- /dev/null +++ b/print.go @@ -0,0 +1,63 @@ +package merkletree + +import ( + "encoding/hex" + "fmt" + + "github.com/fatih/color" +) + +func (mt *MerkleTree) printLevel(parent Hash, iLevel int, maxLevel int) { + for i := 0; i < iLevel; i++ { + fmt.Print(" ") + } + fmt.Print("level ") + fmt.Print(iLevel) + fmt.Print(" - ") + fmt.Print("'" + parent.Hex() + "' = ") + nodeType, _, nodeBytes, err := mt.Get(parent) + if err != nil { + color.Red(err.Error()) + } + var node treeNode + if nodeType == byte(normalNodeType) { + node = parseNodeBytes(nodeBytes) + color.Blue("'" + node.ChildL.Hex() + "' - '" + node.ChildR.Hex() + "'") + } else if nodeType == byte(valueNodeType) { + color.Green("value") + } else if nodeType == byte(finalNodeType) { //typ==FINAL_NODE + fmt.Print("[FinalTree]:") + color.Cyan("final tree node: " + HashBytes(nodeBytes).Hex()) + _, _, leafNodeBytes, err := mt.Get(HashBytes(nodeBytes)) + if err != nil { + color.Red(err.Error()) + } + for i := 0; i < iLevel; i++ { + fmt.Print(" ") + } + color.Cyan(" leaf value: 0x" + hex.EncodeToString(leafNodeBytes)) + } else { + //EMPTY_NODE + fmt.Print("[EmptyBranch]:") + fmt.Println(EmptyNodeValue.Bytes()) + } + iLevel++ + if len(node.ChildR) > 0 && iLevel < maxLevel && nodeType != byte(EmptyNodeType) && nodeType != byte(finalNodeType) { + mt.printLevel(node.ChildL, iLevel, maxLevel) + mt.printLevel(node.ChildR, iLevel, maxLevel) + } +} + +// PrintFullMT prints the tree in the terminal, all the levels with all the nodes +func (mt *MerkleTree) PrintFullMT() { + mt.printLevel(mt.root, 0, mt.numLevels-1) + fmt.Print("root: ") + color.Yellow(mt.Root().Hex()) +} + +// PrintLevelsMT prints the tree in the terminal until a specified depth +func (mt *MerkleTree) PrintLevelsMT(maxLevel int) { + mt.printLevel(mt.root, 0, mt.numLevels-1-maxLevel) + fmt.Print("root: ") + color.Yellow(mt.Root().Hex()) +} diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..8b8a286 --- /dev/null +++ b/utils.go @@ -0,0 +1,61 @@ +package merkletree + +import ( + "encoding/hex" + + "github.com/ethereum/go-ethereum/crypto" +) + +// Hex returns a hex string from the Hash type +func (hash Hash) Hex() string { + r := "0x" + h := hex.EncodeToString(hash[:]) + r = r + h + return r +} + +// Bytes returns a byte array from a Hash +func (hash Hash) Bytes() []byte { + return hash[:] +} + +// HashBytes performs a Keccak256 hash over the bytes +func HashBytes(b []byte) (hash Hash) { + h := crypto.Keccak256(b) + copy(hash[:], h) + return hash +} + +// getPath returns the binary path, from the leaf to the root +func getPath(numLevels int, hi Hash) []bool { + + path := []bool{} + for bitno := numLevels - 2; bitno >= 0; bitno-- { + path = append(path, testbitmap(hi[:], uint(bitno))) + } + return path +} + +func comparePaths(b1 []bool, b2 []bool) int { + for i := len(b1) - 1; i >= 0; i-- { + if b1[i] != b2[i] { + return i + } + } + return -1 +} + +func getEmptiesBetweenIAndPosHash(mt *MerkleTree, iPos int, posHash int) []Hash { + var sibl []Hash + for i := iPos; i >= posHash; i-- { + sibl = append(sibl, EmptyNodeValue) + } + return sibl +} + +func setbitmap(bitmap []byte, bitno uint) { + bitmap[uint(len(bitmap))-bitno/8-1] |= 1 << (bitno % 8) +} +func testbitmap(bitmap []byte, bitno uint) bool { + return bitmap[uint(len(bitmap))-bitno/8-1]&(1<<(bitno%8)) > 0 +} diff --git a/utils_test.go b/utils_test.go new file mode 100644 index 0000000..e2a9352 --- /dev/null +++ b/utils_test.go @@ -0,0 +1,32 @@ +package merkletree + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetSetBitmap(t *testing.T) { + + var v Hash + setbitmap(v[:], 7) + setbitmap(v[:], 8) + setbitmap(v[:], 255) + expected := "0x8000000000000000000000000000000000000000000000000000000000000180" + assert.Equal(t, expected, v.Hex()) + + assert.Equal(t, false, testbitmap(v[:], 6)) + assert.Equal(t, true, testbitmap(v[:], 7)) + assert.Equal(t, true, testbitmap(v[:], 8)) + assert.Equal(t, false, testbitmap(v[:], 9)) + assert.Equal(t, true, testbitmap(v[:], 255)) + +} + +func TestHashBytes(t *testing.T) { + h := HashBytes([]byte("test")).Hex() + assert.Equal(t, "0x9c22ff5f21f0b81b113e63f7db6da94fedef11b2119b4088b89664fb9a3cb658", h) + + h = HashBytes([]byte("authorizeksign")).Hex() + assert.Equal(t, "0x353f867ef725411de05e3d4b0a01c37cf7ad24bcc213141a05ed7726d7932a1f", h) +}