You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

843 lines
21 KiB

3 years ago
3 years ago
3 years ago
  1. /*
  2. Package arbo implements a Merkle Tree compatible with the circomlib
  3. implementation of the MerkleTree (when using the Poseidon hash function),
  4. following the specification from
  5. https://docs.iden3.io/publications/pdfs/Merkle-Tree.pdf and
  6. https://eprint.iacr.org/2018/955.
  7. Also allows to define which hash function to use. So for example, when working
  8. with zkSnarks the Poseidon hash function can be used, but when not, it can be
  9. used the Blake3 hash function, which improves the computation time.
  10. */
  11. package arbo
  12. import (
  13. "bytes"
  14. "encoding/binary"
  15. "encoding/hex"
  16. "fmt"
  17. "io"
  18. "math"
  19. "sync"
  20. "sync/atomic"
  21. "time"
  22. "github.com/iden3/go-merkletree/db"
  23. )
  24. const (
  25. // PrefixValueLen defines the bytes-prefix length used for the Value
  26. // bytes representation stored in the db
  27. PrefixValueLen = 2
  28. // PrefixValueEmpty is used for the first byte of a Value to indicate
  29. // that is an Empty value
  30. PrefixValueEmpty = 0
  31. // PrefixValueLeaf is used for the first byte of a Value to indicate
  32. // that is a Leaf value
  33. PrefixValueLeaf = 1
  34. // PrefixValueIntermediate is used for the first byte of a Value to
  35. // indicate that is a Intermediate value
  36. PrefixValueIntermediate = 2
  37. )
  38. var (
  39. dbKeyRoot = []byte("root")
  40. dbKeyNLeafs = []byte("nleafs")
  41. emptyValue = []byte{0}
  42. )
  43. // Tree defines the struct that implements the MerkleTree functionalities
  44. type Tree struct {
  45. sync.RWMutex
  46. tx db.Tx
  47. db db.Storage
  48. lastAccess int64 // in unix time // TODO delete, is a feature of a upper abstraction level
  49. maxLevels int
  50. root []byte
  51. hashFunction HashFunction
  52. emptyHash []byte
  53. }
  54. // NewTree returns a new Tree, if there is a Tree still in the given storage, it
  55. // will load it.
  56. func NewTree(storage db.Storage, maxLevels int, hash HashFunction) (*Tree, error) {
  57. t := Tree{db: storage, maxLevels: maxLevels, hashFunction: hash}
  58. t.updateAccessTime()
  59. t.emptyHash = make([]byte, t.hashFunction.Len()) // empty
  60. root, err := t.dbGet(dbKeyRoot)
  61. if err == db.ErrNotFound {
  62. // store new root 0
  63. t.tx, err = t.db.NewTx()
  64. if err != nil {
  65. return nil, err
  66. }
  67. t.root = t.emptyHash
  68. if err = t.tx.Put(dbKeyRoot, t.root); err != nil {
  69. return nil, err
  70. }
  71. if err = t.setNLeafs(0); err != nil {
  72. return nil, err
  73. }
  74. if err = t.tx.Commit(); err != nil {
  75. return nil, err
  76. }
  77. return &t, err
  78. } else if err != nil {
  79. return nil, err
  80. }
  81. t.root = root
  82. return &t, nil
  83. }
  84. func (t *Tree) updateAccessTime() {
  85. atomic.StoreInt64(&t.lastAccess, time.Now().Unix())
  86. }
  87. // LastAccess returns the last access timestamp in Unixtime
  88. func (t *Tree) LastAccess() int64 {
  89. return atomic.LoadInt64(&t.lastAccess)
  90. }
  91. // Root returns the root of the Tree
  92. func (t *Tree) Root() []byte {
  93. return t.root
  94. }
  95. // HashFunction returns Tree.hashFunction
  96. func (t *Tree) HashFunction() HashFunction {
  97. return t.hashFunction
  98. }
  99. // Add inserts the key-value into the Tree. If the inputs come from a *big.Int,
  100. // is expected that are represented by a Little-Endian byte array (for circom
  101. // compatibility).
  102. func (t *Tree) Add(k, v []byte) error {
  103. t.updateAccessTime()
  104. t.Lock()
  105. defer t.Unlock()
  106. var err error
  107. t.tx, err = t.db.NewTx()
  108. if err != nil {
  109. return err
  110. }
  111. err = t.add(0, k, v) // add from level 0
  112. if err != nil {
  113. return err
  114. }
  115. // store root to db
  116. if err := t.tx.Put(dbKeyRoot, t.root); err != nil {
  117. return err
  118. }
  119. // update nLeafs
  120. if err = t.incNLeafs(1); err != nil {
  121. return err
  122. }
  123. return t.tx.Commit()
  124. }
  125. func (t *Tree) add(fromLvl int, k, v []byte) error {
  126. // TODO check validity of key & value (for the Tree.HashFunction type)
  127. keyPath := make([]byte, t.hashFunction.Len())
  128. copy(keyPath[:], k)
  129. path := getPath(t.maxLevels, keyPath)
  130. // go down to the leaf
  131. var siblings [][]byte
  132. _, _, siblings, err := t.down(k, t.root, siblings, path, fromLvl, false)
  133. if err != nil {
  134. return err
  135. }
  136. leafKey, leafValue, err := newLeafValue(t.hashFunction, k, v)
  137. if err != nil {
  138. return err
  139. }
  140. if err := t.tx.Put(leafKey, leafValue); err != nil {
  141. return err
  142. }
  143. // go up to the root
  144. if len(siblings) == 0 {
  145. t.root = leafKey
  146. return nil
  147. }
  148. root, err := t.up(leafKey, siblings, path, len(siblings)-1, fromLvl)
  149. if err != nil {
  150. return err
  151. }
  152. t.root = root
  153. return nil
  154. }
  155. // down goes down to the leaf recursively
  156. func (t *Tree) down(newKey, currKey []byte, siblings [][]byte,
  157. path []bool, currLvl int, getLeaf bool) (
  158. []byte, []byte, [][]byte, error) {
  159. if currLvl > t.maxLevels-1 {
  160. return nil, nil, nil, fmt.Errorf("max level")
  161. }
  162. var err error
  163. var currValue []byte
  164. if bytes.Equal(currKey, t.emptyHash) {
  165. // empty value
  166. return currKey, emptyValue, siblings, nil
  167. }
  168. currValue, err = t.dbGet(currKey)
  169. if err != nil {
  170. return nil, nil, nil, err
  171. }
  172. switch currValue[0] {
  173. case PrefixValueEmpty: // empty
  174. // TODO WIP WARNING should not be reached, as the 'if' above should avoid
  175. // reaching this point
  176. // return currKey, empty, siblings, nil
  177. panic("should not be reached, as the 'if' above should avoid reaching this point") // TMP
  178. case PrefixValueLeaf: // leaf
  179. if bytes.Equal(newKey, currKey) {
  180. return nil, nil, nil, fmt.Errorf("key already exists")
  181. }
  182. if !bytes.Equal(currValue, emptyValue) {
  183. if getLeaf {
  184. return currKey, currValue, siblings, nil
  185. }
  186. oldLeafKey, _ := ReadLeafValue(currValue)
  187. oldLeafKeyFull := make([]byte, t.hashFunction.Len())
  188. copy(oldLeafKeyFull[:], oldLeafKey)
  189. // if currKey is already used, go down until paths diverge
  190. oldPath := getPath(t.maxLevels, oldLeafKeyFull)
  191. siblings, err = t.downVirtually(siblings, currKey, newKey, oldPath, path, currLvl)
  192. if err != nil {
  193. return nil, nil, nil, err
  194. }
  195. }
  196. return currKey, currValue, siblings, nil
  197. case PrefixValueIntermediate: // intermediate
  198. if len(currValue) != PrefixValueLen+t.hashFunction.Len()*2 {
  199. return nil, nil, nil,
  200. fmt.Errorf("intermediate value invalid length (expected: %d, actual: %d)",
  201. PrefixValueLen+t.hashFunction.Len()*2, len(currValue))
  202. }
  203. // collect siblings while going down
  204. if path[currLvl] {
  205. // right
  206. lChild, rChild := ReadIntermediateChilds(currValue)
  207. siblings = append(siblings, lChild)
  208. return t.down(newKey, rChild, siblings, path, currLvl+1, getLeaf)
  209. }
  210. // left
  211. lChild, rChild := ReadIntermediateChilds(currValue)
  212. siblings = append(siblings, rChild)
  213. return t.down(newKey, lChild, siblings, path, currLvl+1, getLeaf)
  214. default:
  215. return nil, nil, nil, fmt.Errorf("invalid value")
  216. }
  217. }
  218. // downVirtually is used when in a leaf already exists, and a new leaf which
  219. // shares the path until the existing leaf is being added
  220. func (t *Tree) downVirtually(siblings [][]byte, oldKey, newKey []byte, oldPath,
  221. newPath []bool, currLvl int) ([][]byte, error) {
  222. var err error
  223. if currLvl > t.maxLevels-1 {
  224. return nil, fmt.Errorf("max virtual level %d", currLvl)
  225. }
  226. if oldPath[currLvl] == newPath[currLvl] {
  227. siblings = append(siblings, t.emptyHash)
  228. siblings, err = t.downVirtually(siblings, oldKey, newKey, oldPath, newPath, currLvl+1)
  229. if err != nil {
  230. return nil, err
  231. }
  232. return siblings, nil
  233. }
  234. // reached the divergence
  235. siblings = append(siblings, oldKey)
  236. return siblings, nil
  237. }
  238. // up goes up recursively updating the intermediate nodes
  239. func (t *Tree) up(key []byte, siblings [][]byte, path []bool, currLvl, toLvl int) ([]byte, error) {
  240. var k, v []byte
  241. var err error
  242. if path[currLvl+toLvl] {
  243. k, v, err = newIntermediate(t.hashFunction, siblings[currLvl], key)
  244. if err != nil {
  245. return nil, err
  246. }
  247. } else {
  248. k, v, err = newIntermediate(t.hashFunction, key, siblings[currLvl])
  249. if err != nil {
  250. return nil, err
  251. }
  252. }
  253. // store k-v to db
  254. if err = t.tx.Put(k, v); err != nil {
  255. return nil, err
  256. }
  257. if currLvl == 0 {
  258. // reached the root
  259. return k, nil
  260. }
  261. return t.up(k, siblings, path, currLvl-1, toLvl)
  262. }
  263. func newLeafValue(hashFunc HashFunction, k, v []byte) ([]byte, []byte, error) {
  264. leafKey, err := hashFunc.Hash(k, v, []byte{1})
  265. if err != nil {
  266. return nil, nil, err
  267. }
  268. var leafValue []byte
  269. leafValue = append(leafValue, byte(1))
  270. leafValue = append(leafValue, byte(len(k)))
  271. leafValue = append(leafValue, k...)
  272. leafValue = append(leafValue, v...)
  273. return leafKey, leafValue, nil
  274. }
  275. // ReadLeafValue reads from a byte array the leaf key & value
  276. func ReadLeafValue(b []byte) ([]byte, []byte) {
  277. if len(b) < PrefixValueLen {
  278. return []byte{}, []byte{}
  279. }
  280. kLen := b[1]
  281. if len(b) < PrefixValueLen+int(kLen) {
  282. return []byte{}, []byte{}
  283. }
  284. k := b[PrefixValueLen : PrefixValueLen+kLen]
  285. v := b[PrefixValueLen+kLen:]
  286. return k, v
  287. }
  288. func newIntermediate(hashFunc HashFunction, l, r []byte) ([]byte, []byte, error) {
  289. b := make([]byte, PrefixValueLen+hashFunc.Len()*2)
  290. b[0] = 2
  291. b[1] = byte(len(l))
  292. copy(b[PrefixValueLen:PrefixValueLen+hashFunc.Len()], l)
  293. copy(b[PrefixValueLen+hashFunc.Len():], r)
  294. key, err := hashFunc.Hash(l, r)
  295. if err != nil {
  296. return nil, nil, err
  297. }
  298. return key, b, nil
  299. }
  300. // ReadIntermediateChilds reads from a byte array the two childs keys
  301. func ReadIntermediateChilds(b []byte) ([]byte, []byte) {
  302. if len(b) < PrefixValueLen {
  303. return []byte{}, []byte{}
  304. }
  305. lLen := b[1]
  306. if len(b) < PrefixValueLen+int(lLen) {
  307. return []byte{}, []byte{}
  308. }
  309. l := b[PrefixValueLen : PrefixValueLen+lLen]
  310. r := b[PrefixValueLen+lLen:]
  311. return l, r
  312. }
  313. func getPath(numLevels int, k []byte) []bool {
  314. path := make([]bool, numLevels)
  315. for n := 0; n < numLevels; n++ {
  316. path[n] = k[n/8]&(1<<(n%8)) != 0
  317. }
  318. return path
  319. }
  320. // Update updates the value for a given existing key. If the given key does not
  321. // exist, returns an error.
  322. func (t *Tree) Update(k, v []byte) error {
  323. t.updateAccessTime()
  324. t.Lock()
  325. defer t.Unlock()
  326. var err error
  327. t.tx, err = t.db.NewTx()
  328. if err != nil {
  329. return err
  330. }
  331. keyPath := make([]byte, t.hashFunction.Len())
  332. copy(keyPath[:], k)
  333. path := getPath(t.maxLevels, keyPath)
  334. var siblings [][]byte
  335. _, valueAtBottom, siblings, err := t.down(k, t.root, siblings, path, 0, true)
  336. if err != nil {
  337. return err
  338. }
  339. oldKey, _ := ReadLeafValue(valueAtBottom)
  340. if !bytes.Equal(oldKey, k) {
  341. return fmt.Errorf("key %s does not exist", hex.EncodeToString(k))
  342. }
  343. leafKey, leafValue, err := newLeafValue(t.hashFunction, k, v)
  344. if err != nil {
  345. return err
  346. }
  347. if err := t.tx.Put(leafKey, leafValue); err != nil {
  348. return err
  349. }
  350. // go up to the root
  351. if len(siblings) == 0 {
  352. t.root = leafKey
  353. return t.tx.Commit()
  354. }
  355. root, err := t.up(leafKey, siblings, path, len(siblings)-1, 0)
  356. if err != nil {
  357. return err
  358. }
  359. t.root = root
  360. // store root to db
  361. if err := t.tx.Put(dbKeyRoot, t.root); err != nil {
  362. return err
  363. }
  364. return t.tx.Commit()
  365. }
  366. // GenProof generates a MerkleTree proof for the given key. If the key exists in
  367. // the Tree, the proof will be of existence, if the key does not exist in the
  368. // tree, the proof will be of non-existence.
  369. func (t *Tree) GenProof(k []byte) ([]byte, []byte, error) {
  370. t.updateAccessTime()
  371. keyPath := make([]byte, t.hashFunction.Len())
  372. copy(keyPath[:], k)
  373. path := getPath(t.maxLevels, keyPath)
  374. // go down to the leaf
  375. var siblings [][]byte
  376. _, value, siblings, err := t.down(k, t.root, siblings, path, 0, true)
  377. if err != nil {
  378. return nil, nil, err
  379. }
  380. leafK, leafV := ReadLeafValue(value)
  381. if !bytes.Equal(k, leafK) {
  382. fmt.Println("key not in Tree")
  383. fmt.Println(leafK)
  384. fmt.Println(leafV)
  385. // TODO proof of non-existence
  386. panic(fmt.Errorf("unimplemented"))
  387. }
  388. s := PackSiblings(t.hashFunction, siblings)
  389. return value, s, nil
  390. }
  391. // PackSiblings packs the siblings into a byte array.
  392. // [ 1 byte | L bytes | S * N bytes ]
  393. // [ bitmap length (L) | bitmap | N non-zero siblings ]
  394. // Where the bitmap indicates if the sibling is 0 or a value from the siblings
  395. // array. And S is the size of the output of the hash function used for the
  396. // Tree.
  397. func PackSiblings(hashFunc HashFunction, siblings [][]byte) []byte {
  398. var b []byte
  399. var bitmap []bool
  400. emptySibling := make([]byte, hashFunc.Len())
  401. for i := 0; i < len(siblings); i++ {
  402. if bytes.Equal(siblings[i], emptySibling) {
  403. bitmap = append(bitmap, false)
  404. } else {
  405. bitmap = append(bitmap, true)
  406. b = append(b, siblings[i]...)
  407. }
  408. }
  409. bitmapBytes := bitmapToBytes(bitmap)
  410. l := len(bitmapBytes)
  411. res := make([]byte, l+1+len(b))
  412. res[0] = byte(l) // set the bitmapBytes length
  413. copy(res[1:1+l], bitmapBytes)
  414. copy(res[1+l:], b)
  415. return res
  416. }
  417. // UnpackSiblings unpacks the siblings from a byte array.
  418. func UnpackSiblings(hashFunc HashFunction, b []byte) ([][]byte, error) {
  419. l := b[0]
  420. bitmapBytes := b[1 : 1+l]
  421. bitmap := bytesToBitmap(bitmapBytes)
  422. siblingsBytes := b[1+l:]
  423. iSibl := 0
  424. emptySibl := make([]byte, hashFunc.Len())
  425. var siblings [][]byte
  426. for i := 0; i < len(bitmap); i++ {
  427. if iSibl >= len(siblingsBytes) {
  428. break
  429. }
  430. if bitmap[i] {
  431. siblings = append(siblings, siblingsBytes[iSibl:iSibl+hashFunc.Len()])
  432. iSibl += hashFunc.Len()
  433. } else {
  434. siblings = append(siblings, emptySibl)
  435. }
  436. }
  437. return siblings, nil
  438. }
  439. func bitmapToBytes(bitmap []bool) []byte {
  440. bitmapBytesLen := int(math.Ceil(float64(len(bitmap)) / 8)) //nolint:gomnd
  441. b := make([]byte, bitmapBytesLen)
  442. for i := 0; i < len(bitmap); i++ {
  443. if bitmap[i] {
  444. b[i/8] |= 1 << (i % 8)
  445. }
  446. }
  447. return b
  448. }
  449. func bytesToBitmap(b []byte) []bool {
  450. var bitmap []bool
  451. for i := 0; i < len(b); i++ {
  452. for j := 0; j < 8; j++ {
  453. bitmap = append(bitmap, b[i]&(1<<j) > 0)
  454. }
  455. }
  456. return bitmap
  457. }
  458. // Get returns the value for a given key
  459. func (t *Tree) Get(k []byte) ([]byte, []byte, error) {
  460. keyPath := make([]byte, t.hashFunction.Len())
  461. copy(keyPath[:], k)
  462. path := getPath(t.maxLevels, keyPath)
  463. // go down to the leaf
  464. var siblings [][]byte
  465. _, value, _, err := t.down(k, t.root, siblings, path, 0, true)
  466. if err != nil {
  467. return nil, nil, err
  468. }
  469. leafK, leafV := ReadLeafValue(value)
  470. if !bytes.Equal(k, leafK) {
  471. panic(fmt.Errorf("%s != %s", BytesToBigInt(k), BytesToBigInt(leafK)))
  472. }
  473. return leafK, leafV, nil
  474. }
  475. // CheckProof verifies the given proof. The proof verification depends on the
  476. // HashFunction passed as parameter.
  477. func CheckProof(hashFunc HashFunction, k, v, root, packedSiblings []byte) (bool, error) {
  478. siblings, err := UnpackSiblings(hashFunc, packedSiblings)
  479. if err != nil {
  480. return false, err
  481. }
  482. keyPath := make([]byte, hashFunc.Len())
  483. copy(keyPath[:], k)
  484. key, _, err := newLeafValue(hashFunc, k, v)
  485. if err != nil {
  486. return false, err
  487. }
  488. path := getPath(len(siblings), keyPath)
  489. for i := len(siblings) - 1; i >= 0; i-- {
  490. if path[i] {
  491. key, _, err = newIntermediate(hashFunc, siblings[i], key)
  492. if err != nil {
  493. return false, err
  494. }
  495. } else {
  496. key, _, err = newIntermediate(hashFunc, key, siblings[i])
  497. if err != nil {
  498. return false, err
  499. }
  500. }
  501. }
  502. if bytes.Equal(key[:], root) {
  503. return true, nil
  504. }
  505. return false, nil
  506. }
  507. func (t *Tree) dbGet(k []byte) ([]byte, error) {
  508. // if key is empty, return empty as value
  509. if bytes.Equal(k, t.emptyHash) {
  510. return t.emptyHash, nil
  511. }
  512. v, err := t.db.Get(k)
  513. if err == nil {
  514. return v, nil
  515. }
  516. if t.tx != nil {
  517. return t.tx.Get(k)
  518. }
  519. return nil, db.ErrNotFound
  520. }
  521. // Warning: should be called with a Tree.tx created, and with a Tree.tx.Commit
  522. // after the setNLeafs call.
  523. func (t *Tree) incNLeafs(nLeafs int) error {
  524. oldNLeafs, err := t.GetNLeafs()
  525. if err != nil {
  526. return err
  527. }
  528. newNLeafs := oldNLeafs + nLeafs
  529. return t.setNLeafs(newNLeafs)
  530. }
  531. // Warning: should be called with a Tree.tx created, and with a Tree.tx.Commit
  532. // after the setNLeafs call.
  533. func (t *Tree) setNLeafs(nLeafs int) error {
  534. b := make([]byte, 8)
  535. binary.LittleEndian.PutUint64(b, uint64(nLeafs))
  536. if err := t.tx.Put(dbKeyNLeafs, b); err != nil {
  537. return err
  538. }
  539. return nil
  540. }
  541. // GetNLeafs returns the number of Leafs of the Tree.
  542. func (t *Tree) GetNLeafs() (int, error) {
  543. b, err := t.dbGet(dbKeyNLeafs)
  544. if err != nil {
  545. return 0, err
  546. }
  547. nLeafs := binary.LittleEndian.Uint64(b)
  548. return int(nLeafs), nil
  549. }
  550. // Iterate iterates through the full Tree, executing the given function on each
  551. // node of the Tree.
  552. func (t *Tree) Iterate(f func([]byte, []byte)) error {
  553. // TODO allow to define which root to use
  554. t.updateAccessTime()
  555. return t.iter(t.root, f)
  556. }
  557. // IterateWithStop does the same than Iterate, but with int for the current
  558. // level, and a boolean parameter used by the passed function, is to indicate to
  559. // stop iterating on the branch when the method returns 'true'.
  560. func (t *Tree) IterateWithStop(f func(int, []byte, []byte) bool) error {
  561. t.updateAccessTime()
  562. return t.iterWithStop(t.root, 0, f)
  563. }
  564. func (t *Tree) iterWithStop(k []byte, currLevel int, f func(int, []byte, []byte) bool) error {
  565. v, err := t.dbGet(k)
  566. if err != nil {
  567. return err
  568. }
  569. currLevel++
  570. switch v[0] {
  571. case PrefixValueEmpty:
  572. f(currLevel, k, v)
  573. case PrefixValueLeaf:
  574. f(currLevel, k, v)
  575. case PrefixValueIntermediate:
  576. stop := f(currLevel, k, v)
  577. if stop {
  578. return nil
  579. }
  580. l, r := ReadIntermediateChilds(v)
  581. if err = t.iterWithStop(l, currLevel, f); err != nil {
  582. return err
  583. }
  584. if err = t.iterWithStop(r, currLevel, f); err != nil {
  585. return err
  586. }
  587. default:
  588. return fmt.Errorf("invalid value")
  589. }
  590. return nil
  591. }
  592. func (t *Tree) iter(k []byte, f func([]byte, []byte)) error {
  593. f2 := func(currLvl int, k, v []byte) bool {
  594. f(k, v)
  595. return false
  596. }
  597. return t.iterWithStop(k, 0, f2)
  598. }
  599. // Dump exports all the Tree leafs in a byte array of length:
  600. // [ N * (2+len(k+v)) ]. Where N is the number of key-values, and for each k+v:
  601. // [ 1 byte | 1 byte | S bytes | len(v) bytes ]
  602. // [ len(k) | len(v) | key | value ]
  603. // Where S is the size of the output of the hash function used for the Tree.
  604. func (t *Tree) Dump() ([]byte, error) {
  605. t.updateAccessTime()
  606. // TODO allow to define which root to use
  607. // WARNING current encoding only supports key & values of 255 bytes each
  608. // (due using only 1 byte for the length headers).
  609. var b []byte
  610. err := t.Iterate(func(k, v []byte) {
  611. if v[0] != PrefixValueLeaf {
  612. return
  613. }
  614. leafK, leafV := ReadLeafValue(v)
  615. kv := make([]byte, 2+len(leafK)+len(leafV))
  616. kv[0] = byte(len(leafK))
  617. kv[1] = byte(len(leafV))
  618. copy(kv[2:2+len(leafK)], leafK)
  619. copy(kv[2+len(leafK):], leafV)
  620. b = append(b, kv...)
  621. })
  622. return b, err
  623. }
  624. // ImportDump imports the leafs (that have been exported with the ExportLeafs
  625. // method) in the Tree.
  626. func (t *Tree) ImportDump(b []byte) error {
  627. t.updateAccessTime()
  628. r := bytes.NewReader(b)
  629. count := 0
  630. // TODO instead of adding one by one, use AddBatch (once AddBatch is
  631. // optimized)
  632. var err error
  633. for {
  634. l := make([]byte, 2)
  635. _, err = io.ReadFull(r, l)
  636. if err == io.EOF {
  637. break
  638. } else if err != nil {
  639. return err
  640. }
  641. k := make([]byte, l[0])
  642. _, err = io.ReadFull(r, k)
  643. if err != nil {
  644. return err
  645. }
  646. v := make([]byte, l[1])
  647. _, err = io.ReadFull(r, v)
  648. if err != nil {
  649. return err
  650. }
  651. err = t.Add(k, v)
  652. if err != nil {
  653. return err
  654. }
  655. count++
  656. }
  657. // update nLeafs (once ImportDump uses AddBatch method, this will not be
  658. // needed)
  659. t.tx, err = t.db.NewTx()
  660. if err != nil {
  661. return err
  662. }
  663. if err := t.incNLeafs(count); err != nil {
  664. return err
  665. }
  666. if err = t.tx.Commit(); err != nil {
  667. return err
  668. }
  669. return nil
  670. }
  671. // Graphviz iterates across the full tree to generate a string Graphviz
  672. // representation of the tree and writes it to w
  673. func (t *Tree) Graphviz(w io.Writer, rootKey []byte) error {
  674. return t.GraphvizFirstNLevels(w, rootKey, t.maxLevels)
  675. }
  676. // GraphvizFirstNLevels iterates across the first NLevels of the tree to
  677. // generate a string Graphviz representation of the first NLevels of the tree
  678. // and writes it to w
  679. func (t *Tree) GraphvizFirstNLevels(w io.Writer, rootKey []byte, untilLvl int) error {
  680. fmt.Fprintf(w, `digraph hierarchy {
  681. node [fontname=Monospace,fontsize=10,shape=box]
  682. `)
  683. nChars := 4
  684. nEmpties := 0
  685. err := t.iterWithStop(t.root, 0, func(currLvl int, k, v []byte) bool {
  686. if currLvl == untilLvl {
  687. return true // to stop the iter from going down
  688. }
  689. switch v[0] {
  690. case PrefixValueEmpty:
  691. case PrefixValueLeaf:
  692. fmt.Fprintf(w, "\"%v\" [style=filled];\n", hex.EncodeToString(k[:nChars]))
  693. // key & value from the leaf
  694. kB, vB := ReadLeafValue(v)
  695. fmt.Fprintf(w, "\"%v\" -> {\"k:%v\\nv:%v\"}\n",
  696. hex.EncodeToString(k[:nChars]), hex.EncodeToString(kB[:nChars]),
  697. hex.EncodeToString(vB[:nChars]))
  698. fmt.Fprintf(w, "\"k:%v\\nv:%v\" [style=dashed]\n",
  699. hex.EncodeToString(kB[:nChars]), hex.EncodeToString(vB[:nChars]))
  700. case PrefixValueIntermediate:
  701. l, r := ReadIntermediateChilds(v)
  702. lStr := hex.EncodeToString(l[:nChars])
  703. rStr := hex.EncodeToString(r[:nChars])
  704. eStr := ""
  705. if bytes.Equal(l, t.emptyHash) {
  706. lStr = fmt.Sprintf("empty%v", nEmpties)
  707. eStr += fmt.Sprintf("\"%v\" [style=dashed,label=0];\n",
  708. lStr)
  709. nEmpties++
  710. }
  711. if bytes.Equal(r, t.emptyHash) {
  712. rStr = fmt.Sprintf("empty%v", nEmpties)
  713. eStr += fmt.Sprintf("\"%v\" [style=dashed,label=0];\n",
  714. rStr)
  715. nEmpties++
  716. }
  717. fmt.Fprintf(w, "\"%v\" -> {\"%v\" \"%v\"}\n", hex.EncodeToString(k[:nChars]),
  718. lStr, rStr)
  719. fmt.Fprint(w, eStr)
  720. default:
  721. }
  722. return false
  723. })
  724. fmt.Fprintf(w, "}\n")
  725. return err
  726. }
  727. // PrintGraphviz prints the output of Tree.Graphviz
  728. func (t *Tree) PrintGraphviz(rootKey []byte) error {
  729. return t.PrintGraphvizFirstNLevels(rootKey, t.maxLevels)
  730. }
  731. // PrintGraphvizFirstNLevels prints the output of Tree.GraphvizFirstNLevels
  732. func (t *Tree) PrintGraphvizFirstNLevels(rootKey []byte, untilLvl int) error {
  733. if rootKey == nil {
  734. rootKey = t.Root()
  735. }
  736. w := bytes.NewBufferString("")
  737. fmt.Fprintf(w,
  738. "--------\nGraphviz of the Tree with Root "+hex.EncodeToString(rootKey)+":\n")
  739. err := t.GraphvizFirstNLevels(w, nil, untilLvl)
  740. if err != nil {
  741. fmt.Println(w)
  742. return err
  743. }
  744. fmt.Fprintf(w,
  745. "End of Graphviz of the Tree with Root "+hex.EncodeToString(rootKey)+"\n--------\n")
  746. fmt.Println(w)
  747. return nil
  748. }
  749. // Purge WIP: unimplemented
  750. func (t *Tree) Purge(keys [][]byte) error {
  751. return nil
  752. }